Skip to content

Enterprise agent platform for building, deploying, and operating AI teammates.

79.7 KB
Raw Blame
"""Pin that the orchestrator activates the execution scope on every turn.

The orchestrator resolves the scope and activates it via
``ExecutionScopeContext`` at the same places the acting user is resolved
(``UserContext``): ``execute_task_background`` for normal turns and
``execute_resume_background`` for resumed turns. These tests use a fake
resolver to pin that:

* the resolver is called with the turn's ``task_id`` (as str),
* the resolved scope is active inside the turn's execution context (visible
  to the agent build and the agent run),
* the resumed turn re-resolves the scope (restart/resume correctness), and
* with no resolver registered the turn runs unscoped, exactly as today.
"""

from __future__ import annotations

import asyncio
import logging
import threading
from types import SimpleNamespace
from typing import Any
from unittest.mock import AsyncMock, MagicMock, patch

import pytest
from sqlalchemy.exc import TimeoutError as SQLAlchemyTimeoutError

from tests.shared.execution_scope import register_scope_resolver
from xagent.core.agent.checkpoint import (
    CheckpointAccessRefusedError,
    CheckpointCorruptError,
    CheckpointUnavailableError,
)
from xagent.core.execution_scope import (
    EXECUTION_SCOPE_NOT_PROVIDED,
    ExecutionScope,
    ExecutionScopeAuthorityError,
    get_execution_scope,
    set_execution_scope_snapshot_loader,
)
from xagent.web.api.websocket import _make_command_reply
from xagent.web.models.task import Task, TaskStatus
from xagent.web.models.user import User
from xagent.web.services.client_error_messages import CLIENT_SAFE_TASK_FAILURE
from xagent.web.services.task_command_execution import resume_task
from xagent.web.services.task_execution import (
    ResumeReservationOutcome,
    _acquire_resume_task_lease,
    execute_resume_background,
    execute_task_background,
)
from xagent.web.services.task_lease_service import (
    TaskLease,
    TaskLeaseHeartbeatOutcome,
)
from xagent.web.services.task_settlement_report import SettlementReport


def _make_task_orm() -> Task:
    return Task(
        id=42,
        user_id=1,
        title="scope wiring test",
        description="x",
        status=TaskStatus.RUNNING,
        agent_id=None,
        agent_type="standard",
    )


def _make_user_orm() -> User:
    return User(id=1, username="scope-user", password_hash="hash", is_admin=False)


def _build_db_mock() -> MagicMock:
    """A permissive Session double: any ``query(Model)`` chain resolves to
    the fake Task/User rows above (or None for other models)."""
    rows = {Task: _make_task_orm(), User: _make_user_orm()}

    def _query(model: type) -> Any:
        result = MagicMock()
        result.filter = MagicMock(return_value=result)
        result.first = MagicMock(return_value=rows.get(model))
        result.all = MagicMock(return_value=[])
        result.order_by = MagicMock(return_value=result)
        return result

    db = MagicMock()
    db.query = _query
    return db


def _bg_patches(db: Any) -> list[Any]:
    """Stub persistence so tests observe only scope activation."""

    snapshot = SimpleNamespace(
        task=SimpleNamespace(
            id=42,
            user_id=1,
            status=TaskStatus.RUNNING,
            agent_id=None,
            source=None,
            run_id=None,
        ),
        runtime_user=SimpleNamespace(id=1, is_admin=False),
        agent=None,
        conversation_history=(),
        conversation_watermark=None,
        conversation_event_watermark=None,
        execution_recovery=SimpleNamespace(messages=(), selected_skill_name=None),
    )

    return [
        patch(
            "xagent.web.services.task_setup_snapshot.load_task_setup_snapshot_sync",
            return_value=snapshot,
        ),
        patch(
            "xagent.web.services.task_execution.background_task_manager.wait_for_previous",
            new=AsyncMock(),
        ),
        patch("xagent.web.services.task_execution._register_uploaded_files_for_agent"),
        patch(
            "xagent.web.services.task_execution._finalize_task_execution_result_isolated",
            return_value=SimpleNamespace(
                normalized_outputs=[],
                ai_response="ok",
                chat_response=None,
                waiting_for_control=False,
                terminal_state_committed=True,
                report=SettlementReport(),
                final_task_status=TaskStatus.COMPLETED.value,
                broadcast_meta={
                    "id": 42,
                    "title": "scope wiring test",
                    "description": "x",
                    "execution_mode": None,
                    "updated_at": None,
                },
                late_result=False,
                error_code=None,
                error_details=None,
            ),
        ),
        patch(
            "xagent.web.api.websocket.manager", MagicMock(broadcast_to_task=AsyncMock())
        ),
    ]


@pytest.fixture(autouse=True)
def _stub_routine_history_turn_lookup():
    """Scope tests do not configure persistence for resume provenance."""

    with patch(
        "xagent.web.services.task_execution.load_routine_history_turn_id",
        return_value=None,
    ):
        yield


class _Patches:
    def __init__(self, patches: list[Any]) -> None:
        self._patches = patches

    def __enter__(self) -> None:
        for p in self._patches:
            p.start()

    def __exit__(self, *exc_info: Any) -> None:
        for p in reversed(self._patches):
            p.stop()


@pytest.mark.asyncio
async def test_bg_turn_resolves_and_activates_scope() -> None:
    """The resolver runs at turn start and its scope is active during both
    the agent build (``get_agent_for_task``) and the agent run."""
    scope = ExecutionScope(sandbox_key_suffix="tenant-a")
    resolver_calls: list[str] = []

    def resolver(task_id: str) -> ExecutionScope:
        resolver_calls.append(task_id)
        return scope

    register_scope_resolver(resolver)

    seen: dict[str, Any] = {}
    agent_service = MagicMock()
    agent_service.set_outbound_message_handler = MagicMock()
    agent_service.set_execution_context_messages = MagicMock()
    agent_service.set_recovered_skill_context = MagicMock()

    async def _get_agent_for_task(*args: Any, **kwargs: Any) -> Any:
        seen["scope_at_build"] = get_execution_scope()
        return agent_service

    async def _execute_task(**kwargs: Any) -> dict:
        seen["scope_at_run"] = get_execution_scope()
        return {"success": True, "output": "ok", "status": "completed"}

    agent_manager = MagicMock(
        get_agent_for_task=AsyncMock(side_effect=_get_agent_for_task),
        execute_task=AsyncMock(side_effect=_execute_task),
    )

    with _Patches(_bg_patches(_build_db_mock())):
        await execute_task_background(
            task_id=42,
            user_message="hi",
            context={},
            agent_manager=agent_manager,
            task_owner_user_id=1,
        )

    assert resolver_calls == ["42"]
    assert seen["scope_at_build"] is scope
    assert seen["scope_at_run"] is scope
    # The scope is turn-local: nothing leaks past the turn.
    assert get_execution_scope() is None


@pytest.mark.asyncio
async def test_bg_turn_without_resolver_runs_unscoped() -> None:
    """No resolver registered -> the turn executes unscoped (today's
    behavior, byte-for-byte)."""
    seen: dict[str, Any] = {}
    agent_service = MagicMock()

    async def _execute_task(**kwargs: Any) -> dict:
        seen["scope_at_run"] = get_execution_scope()
        return {"success": True, "output": "ok", "status": "completed"}

    agent_manager = MagicMock(
        get_agent_for_task=AsyncMock(return_value=agent_service),
        execute_task=AsyncMock(side_effect=_execute_task),
    )

    with _Patches(_bg_patches(_build_db_mock())):
        await execute_task_background(
            task_id=42,
            user_message="hi",
            context={},
            agent_manager=agent_manager,
            task_owner_user_id=1,
        )

    assert seen["scope_at_run"] is None


@pytest.mark.asyncio
async def test_bg_turn_explicit_unscoped_does_not_resolve_again() -> None:
    """A caller that already resolved the turn to ``None`` must be able to
    pass that result through without making ``None`` look like "not resolved".
    """

    def unexpected_resolver(task_id: str) -> ExecutionScope:
        raise AssertionError(f"scope for task {task_id} was resolved twice")

    register_scope_resolver(unexpected_resolver)
    agent_service = MagicMock()
    agent_manager = MagicMock(
        get_agent_for_task=AsyncMock(return_value=agent_service),
        execute_task=AsyncMock(
            return_value={"success": True, "output": "ok", "status": "completed"}
        ),
    )

    with _Patches(_bg_patches(_build_db_mock())):
        await execute_task_background(
            task_id=42,
            user_message="hi",
            context={},
            agent_manager=agent_manager,
            task_owner_user_id=1,
            resolved_execution_scope=None,
        )

    assert (
        agent_manager.get_agent_for_task.await_args.kwargs["resolved_execution_scope"]
        is None
    )


@pytest.mark.asyncio
async def test_owned_bg_failure_waits_for_exact_settlement_before_broadcast() -> None:
    """The lease owner must durably settle before any terminal event is sent."""
    agent_service = MagicMock()
    agent_manager = MagicMock(
        get_agent_for_task=AsyncMock(return_value=agent_service),
        execute_task=AsyncMock(side_effect=RuntimeError("owned run failed")),
    )
    broadcast = AsyncMock()
    patches = _bg_patches(_build_db_mock())
    patches[-1] = patch(
        "xagent.web.api.websocket.manager",
        MagicMock(broadcast_to_task=broadcast),
    )

    with _Patches(patches), pytest.raises(RuntimeError, match="owned run failed"):
        await execute_task_background(
            task_id=42,
            user_message="hi",
            context={},
            agent_manager=agent_manager,
            task_owner_user_id=1,
            task_lease=TaskLease(
                task_id=42,
                runner_id="runner-a",
                run_id="run-a",
            ),
            resolved_execution_scope=None,
        )

    broadcast.assert_not_awaited()


@pytest.mark.asyncio
async def test_bg_turn_authority_mismatch_fails_the_turn() -> None:
    """Turn-face consumer: a namespace-affecting disagreement between the
    registered resolver and a persisted snapshot must fail the turn --
    unlike the off-turn consumers (``ManagedFileRef``'s construction, the
    pause/resume handlers), which downgrade the same mismatch to the
    resolver's authoritative answer plus a warning. Being reached off a turn
    is not by itself what decides this: ``websocket._scope_segments_for_task``
    also runs off-turn yet resolves fail-closed, because its caller composes
    a storage key for new bytes."""
    register_scope_resolver(
        lambda task_id: ExecutionScope(sandbox_key_suffix="from-resolver"),
    )
    set_execution_scope_snapshot_loader(
        lambda task_id: ExecutionScope(sandbox_key_suffix="from-snapshot")
    )
    agent_manager = MagicMock()

    with (
        _Patches(_bg_patches(_build_db_mock())),
        pytest.raises(ExecutionScopeAuthorityError),
    ):
        await execute_task_background(
            task_id=42,
            user_message="hi",
            context={},
            agent_manager=agent_manager,
            task_owner_user_id=1,
            task_lease=TaskLease(
                task_id=42,
                runner_id="runner-a",
                run_id="run-a",
            ),
        )

    agent_manager.get_agent_for_task.assert_not_called()


@pytest.mark.asyncio
async def test_bg_turn_resolves_scope_before_loading_snapshot_off_loop() -> None:
    """Scope and setup snapshot are resolved sequentially in workers."""
    main_thread_id = threading.get_ident()
    events: list[tuple[str, int]] = []
    scope = ExecutionScope(sandbox_key_suffix="tenant-a")

    def resolver(task_id: str) -> ExecutionScope:
        events.append(("resolve", threading.get_ident()))
        return scope

    register_scope_resolver(resolver)

    def load_snapshot(*_args: Any, **_kwargs: Any) -> Any:
        events.append(("snapshot", threading.get_ident()))
        return SimpleNamespace(
            task=SimpleNamespace(
                id=42,
                user_id=1,
                status=TaskStatus.RUNNING,
                agent_id=None,
                source=None,
                run_id=None,
            ),
            runtime_user=SimpleNamespace(id=1, is_admin=False),
            agent=None,
            conversation_history=(),
            conversation_watermark=None,
            conversation_event_watermark=None,
            execution_recovery=SimpleNamespace(messages=(), selected_skill_name=None),
        )

    agent_service = MagicMock()
    agent_manager = MagicMock(
        get_agent_for_task=AsyncMock(return_value=agent_service),
        execute_task=AsyncMock(
            return_value={"success": True, "output": "ok", "status": "completed"}
        ),
    )
    patches = _bg_patches(_build_db_mock())
    patches[0] = patch(
        "xagent.web.services.task_setup_snapshot.load_task_setup_snapshot_sync",
        side_effect=load_snapshot,
    )

    with _Patches(patches):
        await execute_task_background(
            task_id=42,
            user_message="hi",
            context={},
            agent_manager=agent_manager,
            task_owner_user_id=1,
        )

    assert events[0][0] == "resolve"
    assert events[0][1] != main_thread_id
    assert events[1][0] == "snapshot"
    assert events[1][1] != main_thread_id


@pytest.mark.asyncio
async def test_resumed_turn_re_resolves_scope() -> None:
    """A resumed execution re-resolves through the hook and runs with the
    identical scope — this is what makes scope survive a process restart:
    nothing is carried in memory, the resolver re-derives it per turn."""
    scope = ExecutionScope(
        sandbox_key_suffix="tenant-a", workspace_segments=("tenant-a",)
    )
    resolver_calls: list[str] = []

    def resolver(task_id: str) -> ExecutionScope:
        resolver_calls.append(task_id)
        return scope

    register_scope_resolver(resolver)

    seen: dict[str, Any] = {}
    agent_service = MagicMock()

    async def _resume(task_id: str, *, metadata: dict[str, Any]) -> dict:
        seen["scope_at_resume"] = get_execution_scope()
        assert metadata == {"task_source": None, "run_id": "scope-run"}
        return {"status": "completed", "success": True, "output": "ok"}

    agent_service.resume_execution_by_id = AsyncMock(side_effect=_resume)

    async def _heartbeat(lease: Any, stop_event: Any) -> None:
        return None

    db = _build_db_mock()

    def _fresh_db_gen():
        yield db

    with _Patches(
        [
            patch(
                "xagent.web.models.database.get_db",
                side_effect=lambda: _fresh_db_gen(),
            ),
            patch(
                "xagent.web.services.task_execution._acquire_resume_task_lease",
                return_value=TaskLease(
                    task_id=42,
                    runner_id="scope-runner",
                    run_id="scope-run",
                ),
            ),
            patch(
                "xagent.web.services.task_execution.run_task_lease_heartbeat",
                side_effect=_heartbeat,
            ),
            patch(
                "xagent.web.services.task_execution.stop_task_lease_heartbeat",
                new=AsyncMock(),
            ),
            patch(
                "xagent.web.services.task_execution._finalize_resumed_task",
                return_value={
                    "task_title": "scope wiring test",
                    "task_description": "x",
                    "task_execution_mode": "flash",
                    "task_agent_id": None,
                    "agent_name": None,
                    "agent_logo_url": None,
                    "final_status": TaskStatus.COMPLETED.value,
                    "lease_released": True,
                    "report": SettlementReport(),
                    "normalized_outputs": [],
                    "output": "ok",
                    "late_result": False,
                },
            ),
            patch(
                "xagent.web.api.websocket.manager",
                MagicMock(broadcast_to_task=AsyncMock()),
            ),
            patch(
                "xagent.web.services.task_execution.background_task_manager.promote_resume_task"
            ),
        ]
    ):
        await execute_resume_background(
            task_id=42,
            agent_service=agent_service,
            task_owner_user_id=1,
        )

    agent_service.resume_execution_by_id.assert_awaited_once()
    assert resolver_calls == ["42"]
    assert seen["scope_at_resume"] is scope
    assert get_execution_scope() is None


@pytest.mark.asyncio
async def test_resume_background_adopts_preacquired_lease_without_reacquiring() -> None:
    lease = TaskLease(task_id=42, runner_id="runner-a", run_id="run-a")
    heartbeat_stop = asyncio.Event()

    async def transferred_heartbeat() -> TaskLeaseHeartbeatOutcome:
        await heartbeat_stop.wait()
        return TaskLeaseHeartbeatOutcome()

    heartbeat_task = asyncio.create_task(transferred_heartbeat())
    agent_service = MagicMock()
    agent_service.resume_execution_by_id = AsyncMock(
        return_value={"status": "completed", "success": True, "output": "ok"}
    )
    acquire = MagicMock()
    start_heartbeat = MagicMock()

    with _Patches(
        [
            patch(
                "xagent.web.services.task_execution._acquire_resume_task_lease",
                new=acquire,
            ),
            patch(
                "xagent.web.services.task_execution.run_task_lease_heartbeat",
                new=start_heartbeat,
            ),
            patch(
                "xagent.web.services.task_execution._finalize_resumed_task",
                return_value={
                    "task_title": "prelease",
                    "task_description": "x",
                    "task_execution_mode": "flash",
                    "task_agent_id": None,
                    "agent_name": None,
                    "agent_logo_url": None,
                    "final_status": TaskStatus.COMPLETED.value,
                    "lease_released": True,
                    "report": SettlementReport(),
                    "normalized_outputs": [],
                    "output": "ok",
                    "late_result": False,
                },
            ),
            patch(
                "xagent.web.api.websocket.manager",
                MagicMock(broadcast_to_task=AsyncMock()),
            ),
            patch(
                "xagent.web.services.task_execution.background_task_manager.promote_resume_task"
            ),
        ]
    ):
        await execute_resume_background(
            task_id=42,
            agent_service=agent_service,
            task_owner_user_id=1,
            expected_run_id="run-a",
            resolved_execution_scope=None,
            preacquired_lease=lease,
            preacquired_heartbeat_stop=heartbeat_stop,
            preacquired_heartbeat_task=heartbeat_task,
        )

    acquire.assert_not_called()
    start_heartbeat.assert_not_called()
    agent_service.resume_execution_by_id.assert_awaited_once_with(
        "42", metadata={"task_source": None, "run_id": "run-a"}
    )
    assert heartbeat_task.done()


@pytest.mark.asyncio
async def test_resumed_task_completed_frame_carries_the_finalized_error_fields() -> (
    None
):
    """The resume frame reads the finalization's code and details.

    The resumed run result carries no ``error_code``; the model-provider
    failure classification only exists on the finalized outcome.
    """
    details = {
        "kind": "access_denied",
        "status_code": 403,
        "provider_code": "provider_code_4204",
        "message": (
            "Model provider call failed (403 provider_code_4204): "
            "Model is decommissioned"
        ),
    }
    lease = TaskLease(task_id=42, runner_id="runner-a", run_id="run-a")
    heartbeat_stop = asyncio.Event()

    async def transferred_heartbeat() -> TaskLeaseHeartbeatOutcome:
        await heartbeat_stop.wait()
        return TaskLeaseHeartbeatOutcome()

    heartbeat_task = asyncio.create_task(transferred_heartbeat())
    agent_service = MagicMock()
    agent_service.resume_execution_by_id = AsyncMock(
        return_value={"status": "failed", "success": False, "output": ""}
    )
    publish = AsyncMock()

    with _Patches(
        [
            patch(
                "xagent.web.services.task_execution._finalize_resumed_task",
                return_value={
                    "task_title": "prelease",
                    "task_description": "x",
                    "task_execution_mode": "flash",
                    "task_agent_id": None,
                    "agent_name": None,
                    "agent_logo_url": None,
                    "final_status": TaskStatus.FAILED.value,
                    "lease_released": True,
                    "report": SettlementReport(),
                    "normalized_outputs": [],
                    "output": "failed",
                    "late_result": False,
                    "error_code": "model_error",
                    "error_details": details,
                },
            ),
            patch(
                "xagent.web.services.task_event_display.publish_task_result",
                new=publish,
            ),
            patch(
                "xagent.web.services.task_execution.background_task_manager.promote_resume_task"
            ),
        ]
    ):
        await execute_resume_background(
            task_id=42,
            agent_service=agent_service,
            task_owner_user_id=1,
            expected_run_id="run-a",
            resolved_execution_scope=None,
            preacquired_lease=lease,
            preacquired_heartbeat_stop=heartbeat_stop,
            preacquired_heartbeat_task=heartbeat_task,
        )

    publish.assert_awaited_once()
    frame = publish.await_args.args[0]
    assert frame["type"] == "task_completed"
    assert frame["error_code"] == "model_error"
    assert frame["error_details"] == details


@pytest.mark.asyncio
async def test_resume_handler_resolves_scope_once_off_loop_for_agent_lookup() -> None:
    """The handler's single off-loop resolution feeds only the agent lookup.

    ``get_agent_for_task`` receives the resolved scope to locate the paused
    task's existing agent. ``execute_resume_background`` is a different
    consumer -- the turn that selects a namespace for new bytes -- so it is
    not handed this same value; it gets ``EXECUTION_SCOPE_NOT_PROVIDED`` and
    resolves for itself.
    """
    main_thread_id = threading.get_ident()
    scope = ExecutionScope(
        sandbox_key_suffix="tenant-a", workspace_segments=("tenant-a",)
    )
    resolver_threads: list[int] = []

    def resolver(task_id: str) -> ExecutionScope:
        assert task_id == "42"
        resolver_threads.append(threading.get_ident())
        return scope

    register_scope_resolver(resolver)
    snapshot = SimpleNamespace(
        task=SimpleNamespace(
            id=42,
            user_id=1,
            status=TaskStatus.PAUSED,
            control_state="paused",
            run_id="run-a",
            state_version=3,
            source=None,
        ),
        runtime_user=SimpleNamespace(id=1, is_admin=False),
    )
    agent_service = MagicMock()
    agent_service.supports_live_control.return_value = True
    agent_manager = MagicMock(get_agent_for_task=AsyncMock(return_value=agent_service))
    resume_started = asyncio.Event()
    resume_kwargs: dict[str, Any] = {}

    async def execute_resume(**kwargs: Any) -> None:
        resume_kwargs.update(kwargs)
        resume_started.set()

    background_manager = MagicMock()
    background_manager.running_tasks = {}
    background_manager.resume_admission_state.return_value = None
    background_manager.try_reserve_resume.return_value = (
        ResumeReservationOutcome.RESERVED
    )
    transition = AsyncMock(
        return_value=SimpleNamespace(run_id="run-a", status=TaskStatus.PAUSED)
    )

    with _Patches(
        [
            patch(
                "xagent.web.services.task_setup_snapshot.load_task_setup_snapshot_sync",
                return_value=snapshot,
            ),
            patch(
                "xagent.web.services.agent_service_manager.get_agent_manager",
                return_value=agent_manager,
            ),
            patch(
                "xagent.web.api.websocket.task_execution_controller.transition",
                new=transition,
            ),
            patch(
                "xagent.web.services.task_execution.background_task_manager",
                background_manager,
            ),
            # The handler asks the DB whether another process still holds a
            # live lease before scheduling; this suite drives it without a
            # task row, so answer "no foreign owner" explicitly.
            patch(
                "xagent.web.services.task_command_execution.task_has_live_foreign_runner",
                return_value=False,
            ),
            patch(
                "xagent.web.services.task_execution.execute_resume_background",
                side_effect=execute_resume,
            ),
        ]
    ):
        await resume_task(
            _make_command_reply(MagicMock()),
            42,
            {"user": SimpleNamespace(id=1, is_admin=False)},
        )
        await asyncio.wait_for(resume_started.wait(), timeout=1)

    assert len(resolver_threads) == 1
    assert resolver_threads[0] != main_thread_id
    assert (
        agent_manager.get_agent_for_task.await_args.kwargs["resolved_execution_scope"]
        is scope
    )
    assert resume_kwargs["resolved_execution_scope"] is EXECUTION_SCOPE_NOT_PROVIDED


@pytest.mark.asyncio
async def test_resume_task_forwards_the_task_rows_source() -> None:
    """``resume_task`` must forward the paused task row's own ``source`` as
    ``trusted_task_source`` -- not leave it at the ``execute_resume_background``
    default of ``None``.

    ``task_source`` is the key the MCP approval gate selects a registration
    by; a resumed run that lost it would have its checkpointed source
    overwritten by the runner's overlay reading nothing, silently ungating
    (or wrongly refusing) an approval the user is resuming. No existing test
    asserted this kwarg at all: a deleted ``trusted_task_source=`` line at
    this call site would still pass the rest of the suite.
    """
    register_scope_resolver(lambda task_id: None)
    snapshot = SimpleNamespace(
        task=SimpleNamespace(
            id=42,
            user_id=1,
            status=TaskStatus.PAUSED,
            control_state="paused",
            run_id="run-a",
            state_version=3,
            source="toby-slack",
        ),
        runtime_user=SimpleNamespace(id=1, is_admin=False),
    )
    agent_service = MagicMock()
    agent_service.supports_live_control.return_value = True
    agent_manager = MagicMock(get_agent_for_task=AsyncMock(return_value=agent_service))
    resume_started = asyncio.Event()
    resume_kwargs: dict[str, Any] = {}

    async def execute_resume(**kwargs: Any) -> None:
        resume_kwargs.update(kwargs)
        resume_started.set()

    background_manager = MagicMock()
    background_manager.running_tasks = {}
    background_manager.resume_admission_state.return_value = None
    background_manager.try_reserve_resume.return_value = (
        ResumeReservationOutcome.RESERVED
    )
    transition = AsyncMock(
        return_value=SimpleNamespace(run_id="run-a", status=TaskStatus.PAUSED)
    )

    with _Patches(
        [
            patch(
                "xagent.web.services.task_setup_snapshot.load_task_setup_snapshot_sync",
                return_value=snapshot,
            ),
            patch(
                "xagent.web.services.agent_service_manager.get_agent_manager",
                return_value=agent_manager,
            ),
            patch(
                "xagent.web.api.websocket.task_execution_controller.transition",
                new=transition,
            ),
            patch(
                "xagent.web.services.task_execution.background_task_manager",
                background_manager,
            ),
            patch(
                "xagent.web.services.task_command_execution.task_has_live_foreign_runner",
                return_value=False,
            ),
            patch(
                "xagent.web.services.task_execution.execute_resume_background",
                side_effect=execute_resume,
            ),
        ]
    ):
        await resume_task(
            _make_command_reply(MagicMock()),
            42,
            {"user": SimpleNamespace(id=1, is_admin=False)},
        )
        await asyncio.wait_for(resume_started.wait(), timeout=1)

    assert resume_kwargs["trusted_task_source"] == "toby-slack"


@pytest.mark.asyncio
async def test_resume_survives_a_scope_authority_mismatch() -> None:
    """A control operation must stay available while the scope is in dispute.

    Resume locates an already-established workspace and sandbox for a task
    that already exists using the downgraded (never-raising) off-turn
    resolution -- that lookup selects no namespace for new bytes, so it must
    not be blocked by the dispute. The turn it hands off to is a different
    consumer that does select a namespace for new bytes, so the handler must
    not pass its own downgraded answer through: it leaves that argument at
    ``EXECUTION_SCOPE_NOT_PROVIDED`` so the scheduled turn performs its own
    fail-closed resolution (see
    ``test_resume_background_fails_closed_on_scope_authority_mismatch`` for
    that resolution actually raising). Failing the handler itself instead
    would leave the user unable to resume, pause or stop the task at all,
    with the authority error escaping the socket loop.
    """
    resolver_scope = ExecutionScope(
        sandbox_key_suffix="from-resolver", workspace_segments=("from-resolver",)
    )
    register_scope_resolver(lambda task_id: resolver_scope)
    set_execution_scope_snapshot_loader(
        lambda task_id: ExecutionScope(
            sandbox_key_suffix="from-snapshot", workspace_segments=("from-snapshot",)
        )
    )
    snapshot = SimpleNamespace(
        task=SimpleNamespace(
            id=42,
            user_id=1,
            status=TaskStatus.PAUSED,
            control_state="paused",
            run_id="run-a",
            state_version=3,
            source=None,
        ),
        runtime_user=SimpleNamespace(id=1, is_admin=False),
    )
    agent_service = MagicMock()
    agent_service.supports_live_control.return_value = True
    agent_manager = MagicMock(get_agent_for_task=AsyncMock(return_value=agent_service))
    resume_started = asyncio.Event()
    resume_kwargs: dict[str, Any] = {}

    async def execute_resume(**kwargs: Any) -> None:
        resume_kwargs.update(kwargs)
        resume_started.set()

    background_manager = MagicMock()
    background_manager.running_tasks = {}
    background_manager.resume_admission_state.return_value = None
    background_manager.try_reserve_resume.return_value = (
        ResumeReservationOutcome.RESERVED
    )

    with _Patches(
        [
            patch(
                "xagent.web.services.task_setup_snapshot.load_task_setup_snapshot_sync",
                return_value=snapshot,
            ),
            patch(
                "xagent.web.services.agent_service_manager.get_agent_manager",
                return_value=agent_manager,
            ),
            patch(
                "xagent.web.api.websocket.task_execution_controller.transition",
                new=AsyncMock(
                    return_value=SimpleNamespace(
                        run_id="run-a", status=TaskStatus.PAUSED
                    )
                ),
            ),
            patch(
                "xagent.web.services.task_execution.background_task_manager",
                background_manager,
            ),
            # The handler asks the DB whether another process still holds a
            # live lease before scheduling; this suite drives it without a
            # task row, so answer "no foreign owner" explicitly.
            patch(
                "xagent.web.services.task_command_execution.task_has_live_foreign_runner",
                return_value=False,
            ),
            patch(
                "xagent.web.services.task_execution.execute_resume_background",
                side_effect=execute_resume,
            ),
        ]
    ):
        await resume_task(
            _make_command_reply(MagicMock()),
            42,
            {"user": SimpleNamespace(id=1, is_admin=False)},
        )
        await asyncio.wait_for(resume_started.wait(), timeout=1)

    # The off-turn lookup that locates the existing agent still uses the
    # downgraded resolver answer -- a control operation on an existing task
    # must not be blocked by the dispute.
    assert (
        agent_manager.get_agent_for_task.await_args.kwargs["resolved_execution_scope"]
        is resolver_scope
    )
    # The scheduled turn is a different consumer: the handler must not pass
    # its own downgraded answer through, so the turn can run its own
    # fail-closed resolution instead of silently selecting the resolver's
    # disputed answer for new bytes.
    assert resume_kwargs["resolved_execution_scope"] is EXECUTION_SCOPE_NOT_PROVIDED


@pytest.mark.asyncio
async def test_resume_background_fails_closed_on_scope_authority_mismatch(
    caplog: pytest.LogCaptureFixture,
) -> None:
    """The scheduled turn's own re-resolution is fail-closed, not downgraded.

    ``execute_resume_background`` is the consumer that selects a namespace
    for new bytes for the resumed turn. Unlike the handler's off-turn lookup
    pinned in ``test_resume_survives_a_scope_authority_mismatch``, a
    resolver/snapshot disagreement reaching this function's own resolution
    (``resolved_execution_scope`` left at ``EXECUTION_SCOPE_NOT_PROVIDED``)
    must fail the turn instead of silently proceeding under either answer.
    The ``ExecutionScopeAuthorityError`` raised inside this background task
    is caught by its own settlement handler (a background task has no
    caller to propagate to) and converted into a broadcast task-failure
    event rather than executing the resume -- that conversion, not an
    escaping exception, is what "fails closed" means for this consumer.
    """
    register_scope_resolver(
        lambda task_id: ExecutionScope(sandbox_key_suffix="from-resolver")
    )
    set_execution_scope_snapshot_loader(
        lambda task_id: ExecutionScope(sandbox_key_suffix="from-snapshot")
    )
    agent_service = MagicMock()
    agent_service.resume_execution_by_id = AsyncMock()
    broadcast_manager = MagicMock(broadcast_to_task=AsyncMock())

    with _Patches(
        [
            patch(
                "xagent.web.services.task_execution.background_task_manager.promote_resume_task"
            ),
            patch("xagent.web.api.websocket.manager", broadcast_manager),
        ]
    ):
        await execute_resume_background(
            task_id=42,
            agent_service=agent_service,
            task_owner_user_id=1,
        )

    agent_service.resume_execution_by_id.assert_not_awaited()
    broadcast_manager.broadcast_to_task.assert_awaited_once()
    broadcast_args, _ = broadcast_manager.broadcast_to_task.await_args
    event, broadcast_task_id = broadcast_args
    assert broadcast_task_id == 42
    assert event["type"] == "task_error"
    assert event["task"]["status"] == TaskStatus.FAILED.value
    assert event["message"] == CLIENT_SAFE_TASK_FAILURE
    assert event["error"] == CLIENT_SAFE_TASK_FAILURE
    assert "execution scope authority mismatch" not in repr(event).lower()
    assert "execution scope authority mismatch" in caplog.text.lower()


def test_resume_acquire_checkout_timeout_before_claim_does_not_try_cleanup() -> None:
    query = MagicMock()
    query.filter.return_value = query
    query.first.side_effect = SQLAlchemyTimeoutError("pool exhausted")
    db = MagicMock()
    db.query.return_value = query
    session_context = MagicMock()
    session_context.__enter__.return_value = db
    session_context.__exit__.return_value = False
    session_factory = MagicMock(return_value=session_context)
    transaction_claim = MagicMock()

    with (
        patch(
            "xagent.web.services.task_execution.get_session_local",
            return_value=session_factory,
        ),
        patch(
            "xagent.web.services.task_execution.acquire_task_lease_no_commit",
            transaction_claim,
        ),
        pytest.raises(SQLAlchemyTimeoutError, match="pool exhausted"),
    ):
        _acquire_resume_task_lease(42, 1, "run-a")

    transaction_claim.assert_not_called()


@pytest.mark.asyncio
async def test_resume_db_lifecycle_runs_in_short_session_workers() -> None:
    """Resume lease acquisition, terminal persistence, and fallback release
    must all run off-loop, while the tracker is constructed without a caller
    Session and the resolved unscoped value stays explicit.
    """
    main_thread_id = threading.get_ident()
    worker_events: list[tuple[str, int]] = []
    lease = TaskLease(task_id=42, runner_id="runner-a", run_id="run-a")

    def acquire_resume_lease(*args: Any, **kwargs: Any) -> object:
        worker_events.append(("acquire", threading.get_ident()))
        return lease

    def finalize_resume(*args: Any, **kwargs: Any) -> dict[str, Any]:
        worker_events.append(("finalize", threading.get_ident()))
        return {
            "task_title": "scope wiring test",
            "task_description": "x",
            "task_execution_mode": "flash",
            "task_agent_id": None,
            "agent_name": None,
            "agent_logo_url": None,
            "final_status": TaskStatus.COMPLETED.value,
            "lease_released": False,
            "report": SettlementReport(),
            "normalized_outputs": [],
            "output": "ok",
            "late_result": False,
        }

    def release_resume_lease(
        acquired_lease: object,
        *,
        error_message: str | None,
        injection_outcome_unknown: bool = False,
        interruption: Any = None,
        report: SettlementReport | None = None,
        turn_id: str | None = None,
        classify_unknown_tool_effect: bool = False,
    ) -> None:
        assert acquired_lease is lease
        assert error_message is None
        assert report == SettlementReport()
        assert interruption is None
        assert classify_unknown_tool_effect is False
        assert turn_id is None
        worker_events.append(("release", threading.get_ident()))

    class FakeTracker:
        quota_interrupt_reason = None

        def __init__(self, *, task_id: int, **kwargs: Any) -> None:
            assert task_id == 42
            assert kwargs == {
                "expected_run_id": "run-a",
                "expected_runner_id": "runner-a",
                "expected_attempt_id": None,
            }

        async def start_tracking(self) -> None:
            return None

        async def complete_tracking(self) -> None:
            return None

        async def interrupt_reason_for_quota(self) -> None:
            return None

    agent_service = MagicMock()
    agent_service.resume_execution_by_id = AsyncMock(
        return_value={"status": "completed", "success": True, "output": "ok"}
    )

    with _Patches(
        [
            patch(
                "xagent.web.services.task_execution._acquire_resume_task_lease",
                side_effect=acquire_resume_lease,
                create=True,
            ),
            patch(
                "xagent.web.services.task_execution._finalize_resumed_task",
                side_effect=finalize_resume,
                create=True,
            ),
            patch(
                "xagent.web.services.task_execution._settle_resumed_task_lease",
                side_effect=release_resume_lease,
                create=True,
            ),
            patch(
                "xagent.web.services.task_execution.run_task_lease_heartbeat",
                new=AsyncMock(),
            ),
            patch(
                "xagent.web.services.task_execution.stop_task_lease_heartbeat",
                new=AsyncMock(),
            ),
            patch(
                "xagent.web.services.task_execution.load_routine_history_turn_id",
                return_value=None,
            ),
            patch(
                "xagent.web.api.websocket.manager",
                MagicMock(broadcast_to_task=AsyncMock()),
            ),
            patch(
                "xagent.web.services.task_execution.background_task_manager.promote_resume_task"
            ),
            patch("xagent.web.tracking.task_tracker.TaskTracker", FakeTracker),
        ]
    ):
        await execute_resume_background(
            task_id=42,
            agent_service=agent_service,
            task_owner_user_id=1,
            resolved_execution_scope=None,
        )

    assert [name for name, _thread_id in worker_events] == [
        "acquire",
        "finalize",
        "release",
    ]
    assert all(thread_id != main_thread_id for _, thread_id in worker_events)


@pytest.mark.asyncio
async def test_resume_final_usage_and_heartbeat_finish_before_lease_release() -> None:
    lease = TaskLease(task_id=42, runner_id="runner-a", run_id="run-a")
    events: list[str] = []

    class FakeTracker:
        quota_interrupt_reason = None

        def __init__(self, *, task_id: int, **kwargs: Any) -> None:
            assert task_id == 42
            assert kwargs == {
                "expected_run_id": "run-a",
                "expected_runner_id": "runner-a",
                "expected_attempt_id": None,
            }

        async def start_tracking(self) -> None:
            return None

        async def complete_tracking(self) -> None:
            events.append("usage")

        async def interrupt_reason_for_quota(self) -> None:
            return None

    async def stop_heartbeat(*_args: Any) -> None:
        events.append("heartbeat")

    def finalize(*_args: Any, **_kwargs: Any) -> dict[str, Any]:
        events.append("lease-finalizer")
        return {
            "task_title": "scope wiring test",
            "task_description": "x",
            "task_execution_mode": "flash",
            "task_agent_id": None,
            "agent_name": None,
            "agent_logo_url": None,
            "final_status": TaskStatus.COMPLETED.value,
            "lease_released": True,
            "report": SettlementReport(),
            "normalized_outputs": [],
            "output": "ok",
            "late_result": False,
        }

    agent_service = MagicMock()
    agent_service.resume_execution_by_id = AsyncMock(
        return_value={"status": "completed", "success": True, "output": "ok"}
    )

    with _Patches(
        [
            patch(
                "xagent.web.services.task_execution._acquire_resume_task_lease",
                return_value=lease,
            ),
            patch(
                "xagent.web.services.task_execution.run_task_lease_heartbeat",
                new=AsyncMock(),
            ),
            patch(
                "xagent.web.services.task_execution.stop_task_lease_heartbeat",
                side_effect=stop_heartbeat,
            ),
            patch(
                "xagent.web.services.task_execution._finalize_resumed_task",
                side_effect=finalize,
            ),
            patch(
                "xagent.web.api.websocket.manager",
                MagicMock(broadcast_to_task=AsyncMock()),
            ),
            patch(
                "xagent.web.services.task_execution.background_task_manager.promote_resume_task"
            ),
            patch("xagent.web.tracking.task_tracker.TaskTracker", FakeTracker),
        ]
    ):
        await execute_resume_background(
            task_id=42,
            agent_service=agent_service,
            task_owner_user_id=1,
            resolved_execution_scope=None,
        )

    assert events == ["usage", "heartbeat", "lease-finalizer"]


@pytest.mark.asyncio
async def test_resume_pool_timeout_does_not_start_secondary_db_cleanup(caplog) -> None:
    """A resume checkout timeout retains its exact lease for TTL recovery."""
    lease = TaskLease(task_id=42, runner_id="runner-a", run_id="run-a")
    mark_delivery = MagicMock()
    settle = MagicMock()
    snapshot = AsyncMock()
    tracker_instances: list[Any] = []

    class FakeTracker:
        quota_interrupt_reason = None

        def __init__(self, *, task_id: int, **kwargs: Any) -> None:
            assert task_id == 42
            assert kwargs == {
                "expected_run_id": "run-a",
                "expected_runner_id": "runner-a",
                "expected_attempt_id": None,
            }
            self.complete_tracking = AsyncMock()
            self.stop_periodic_updates = AsyncMock()
            tracker_instances.append(self)

        async def start_tracking(self) -> None:
            return None

        async def interrupt_reason_for_quota(self) -> None:
            return None

    agent_service = MagicMock()
    agent_service.resume_execution_by_id = AsyncMock(
        return_value={"status": "completed", "success": True, "output": "ok"}
    )
    ws_manager = MagicMock(broadcast_to_task=AsyncMock())
    caplog.set_level(logging.ERROR, logger="xagent.web.services.task_execution")

    with _Patches(
        [
            patch(
                "xagent.web.services.task_execution._acquire_resume_task_lease",
                return_value=lease,
            ),
            patch(
                "xagent.web.services.task_execution._finalize_resumed_task",
                side_effect=SQLAlchemyTimeoutError("pool exhausted"),
            ),
            patch(
                "xagent.web.services.task_execution._settle_resumed_task_lease",
                settle,
            ),
            patch(
                "xagent.web.services.task_execution.mark_user_message_delivery_sync",
                mark_delivery,
            ),
            patch(
                "xagent.web.api.websocket.task_execution_controller.snapshot",
                new=snapshot,
            ),
            patch(
                "xagent.web.services.task_execution.run_task_lease_heartbeat",
                new=AsyncMock(),
            ),
            patch(
                "xagent.web.services.task_execution.stop_task_lease_heartbeat",
                new=AsyncMock(),
            ),
            patch("xagent.web.api.websocket.manager", ws_manager),
            patch(
                "xagent.web.services.task_execution.background_task_manager.promote_resume_task"
            ),
            patch("xagent.web.tracking.task_tracker.TaskTracker", FakeTracker),
        ]
    ):
        await execute_resume_background(
            task_id=42,
            agent_service=agent_service,
            task_owner_user_id=1,
            delivery_turn_id="resume-turn",
            expected_run_id="run-a",
            resolved_execution_scope=None,
        )

    assert tracker_instances
    tracker_instances[0].complete_tracking.assert_awaited_once()
    tracker_instances[0].stop_periodic_updates.assert_not_awaited()
    mark_delivery.assert_not_called()
    snapshot.assert_not_awaited()
    settle.assert_not_called()
    assert "task_id=42" in caplog.text
    assert "component=resume" in caplog.text
    assert "retaining lease for TTL recovery" in caplog.text
    assert not any(
        call.args[0].get("type") == "task_error"
        for call in ws_manager.broadcast_to_task.call_args_list
    )


def test_acquire_resume_lease_reports_prior_status_before_claiming() -> None:
    """The real acquire must fill the prior-status box on the claim path.

    Every restore-to-prior test below patches this function with a fake
    that mirrors the out parameter, so nothing else would notice if the
    production function stopped populating it -- the resume path would
    just silently fall back to the terminal FAILED branch. Pin the real
    function against its own contract, and pin that the status recorded
    is the pre-acquisition one rather than the RUNNING the claim writes.
    """
    lease = TaskLease(task_id=42, runner_id="runner-a", run_id="run-a")
    task = SimpleNamespace(id=42, user_id=1, status=TaskStatus.WAITING_FOR_USER)
    query = MagicMock()
    query.filter.return_value = query
    query.first.return_value = task
    db = MagicMock()
    db.query.return_value = query
    session_context = MagicMock()
    session_context.__enter__.return_value = db
    session_context.__exit__.return_value = False
    session_factory = MagicMock(return_value=session_context)

    prior_status_box: list[TaskStatus] = []
    with (
        patch(
            "xagent.web.services.task_execution.get_session_local",
            return_value=session_factory,
        ),
        patch(
            "xagent.web.services.task_execution.acquire_task_lease_no_commit",
            return_value=lease,
        ),
        patch("xagent.web.services.task_execution.sync_workforce_run_status"),
    ):
        acquired = _acquire_resume_task_lease(
            42,
            1,
            "run-a",
            prior_status_out=prior_status_box,
        )

    assert acquired is lease
    assert prior_status_box == [TaskStatus.WAITING_FOR_USER]


def test_acquire_resume_lease_reports_prior_status_when_claim_is_lost() -> None:
    """A lost claim must not leave the box holding a fabricated status."""
    task = SimpleNamespace(id=42, user_id=1, status=TaskStatus.PAUSED)
    query = MagicMock()
    query.filter.return_value = query
    query.first.return_value = task
    db = MagicMock()
    db.query.return_value = query
    session_context = MagicMock()
    session_context.__enter__.return_value = db
    session_context.__exit__.return_value = False
    session_factory = MagicMock(return_value=session_context)

    prior_status_box: list[TaskStatus] = []
    with (
        patch(
            "xagent.web.services.task_execution.get_session_local",
            return_value=session_factory,
        ),
        patch(
            "xagent.web.services.task_execution.acquire_task_lease_no_commit",
            return_value=None,
        ),
    ):
        acquired = _acquire_resume_task_lease(
            42,
            1,
            "run-a",
            prior_status_out=prior_status_box,
        )

    assert acquired is None
    # Recorded, but the caller only consults it when a lease was claimed,
    # so a lost claim can never restore a status it does not own.
    assert prior_status_box == [TaskStatus.PAUSED]


@pytest.mark.asyncio
async def test_resume_background_restores_prior_status_for_preacquired_lease() -> None:
    """A resume that adopts someone else's lease owes the same restore.

    The A2A input-required flow claims the lease itself and hands it over,
    so this entry never runs the acquisition that captures the status. If
    the handover does not carry it, a checkpoint the resume cannot read
    downgrades a still-resumable task to a terminal FAILED on that path
    while the self-acquiring path recovers it.
    """
    lease = TaskLease(task_id=42, runner_id="runner-a", run_id="run-a")
    heartbeat_stop = asyncio.Event()
    heartbeat_task: asyncio.Task[Any] = asyncio.create_task(asyncio.sleep(0))
    settle = MagicMock()
    restore = MagicMock(return_value=True)

    class FakeTracker:
        quota_interrupt_reason = None

        def __init__(self, *, task_id: int, **kwargs: Any) -> None:
            self.complete_tracking = AsyncMock()
            self.stop_periodic_updates = AsyncMock()

        async def start_tracking(self) -> None:
            return None

        async def interrupt_reason_for_quota(self) -> None:
            return None

    agent_service = MagicMock()
    agent_service.resume_execution_by_id = AsyncMock(
        side_effect=CheckpointUnavailableError("checkpoint query failed")
    )
    ws_manager = MagicMock(broadcast_to_task=AsyncMock())

    with _Patches(
        [
            patch(
                "xagent.web.services.task_execution._settle_resumed_task_lease", settle
            ),
            patch(
                "xagent.web.services.task_execution._restore_resumed_task_lease_to_prior_status",
                restore,
            ),
            patch(
                "xagent.web.services.task_execution.stop_task_lease_heartbeat",
                new=AsyncMock(),
            ),
            patch("xagent.web.api.websocket.manager", ws_manager),
            patch(
                "xagent.web.services.task_execution.background_task_manager.promote_resume_task"
            ),
            patch("xagent.web.tracking.task_tracker.TaskTracker", FakeTracker),
        ]
    ):
        await execute_resume_background(
            task_id=42,
            agent_service=agent_service,
            task_owner_user_id=1,
            expected_run_id="run-a",
            resolved_execution_scope=None,
            preacquired_lease=lease,
            preacquired_heartbeat_stop=heartbeat_stop,
            preacquired_heartbeat_task=heartbeat_task,
            preacquired_prior_status=TaskStatus.WAITING_FOR_USER,
        )

    settle.assert_not_called()
    restore.assert_called_once()
    assert restore.call_args.kwargs["status"] == TaskStatus.WAITING_FOR_USER
    assert not any(
        call.args[0].get("type") == "task_error"
        for call in ws_manager.broadcast_to_task.call_args_list
    )


def _fake_acquire_with_prior_status(lease: TaskLease, prior_status: TaskStatus):
    """Mirror ``_acquire_resume_task_lease``'s prior-status side channel."""

    def _acquire(
        task_id_arg: int,
        task_owner_user_id_arg: int | None,
        expected_run_id_arg: str | None,
        *,
        prior_status_out: list[Any] | None = None,
        refuse_terminal_status: bool = False,
        run_not_resumable_out: list[bool] | None = None,
        ended_status_out: list[Any] | None = None,
        history_turn_id_out: list[str | None] | None = None,
    ) -> TaskLease:
        if prior_status_out is not None:
            prior_status_out.append(prior_status)
        return lease

    return _acquire


@pytest.mark.asyncio
@pytest.mark.parametrize(
    "read_error",
    [
        CheckpointUnavailableError("checkpoint store unavailable"),
        CheckpointAccessRefusedError("partition refused"),
    ],
)
async def test_resume_background_restores_prior_status_on_checkpoint_read_failure(
    read_error: Exception,
) -> None:
    """A retryable checkpoint read failure during resume (not a pool
    timeout) must restore the task to whatever it was before this attempt
    claimed the lease, not fall through to the generic terminal FAILED
    path. Unavailable and refused reads share the restore branch."""
    lease = TaskLease(task_id=42, runner_id="runner-a", run_id="run-a")
    settle = MagicMock()
    restore = MagicMock(return_value=True)
    mark_delivery = MagicMock()

    class FakeTracker:
        quota_interrupt_reason = None

        def __init__(self, *, task_id: int, **kwargs: Any) -> None:
            self.complete_tracking = AsyncMock()
            self.stop_periodic_updates = AsyncMock()

        async def start_tracking(self) -> None:
            return None

        async def interrupt_reason_for_quota(self) -> None:
            return None

    agent_service = MagicMock()
    agent_service.resume_execution_by_id = AsyncMock(side_effect=read_error)
    ws_manager = MagicMock(broadcast_to_task=AsyncMock())

    with _Patches(
        [
            patch(
                "xagent.web.services.task_execution._acquire_resume_task_lease",
                side_effect=_fake_acquire_with_prior_status(
                    lease, TaskStatus.WAITING_FOR_USER
                ),
            ),
            patch(
                "xagent.web.services.task_execution._settle_resumed_task_lease", settle
            ),
            patch(
                "xagent.web.services.task_execution._restore_resumed_task_lease_to_prior_status",
                restore,
            ),
            patch(
                "xagent.web.services.task_execution.mark_user_message_delivery_sync",
                mark_delivery,
            ),
            patch(
                "xagent.web.services.task_execution.run_task_lease_heartbeat",
                new=AsyncMock(),
            ),
            patch(
                "xagent.web.services.task_execution.stop_task_lease_heartbeat",
                new=AsyncMock(),
            ),
            patch("xagent.web.api.websocket.manager", ws_manager),
            patch(
                "xagent.web.services.task_execution.background_task_manager.promote_resume_task"
            ),
            patch("xagent.web.tracking.task_tracker.TaskTracker", FakeTracker),
        ]
    ):
        await execute_resume_background(
            task_id=42,
            agent_service=agent_service,
            task_owner_user_id=1,
            delivery_turn_id="resume-turn",
            expected_run_id="run-a",
            resolved_execution_scope=None,
        )

    settle.assert_not_called()
    restore.assert_called_once()
    _, restore_kwargs = restore.call_args
    assert restore.call_args.args[0] == lease
    assert restore_kwargs["status"] == TaskStatus.WAITING_FOR_USER
    # Retryable, not a rejected/failed delivery: the client should retry
    # with a fresh id once the task is back to waiting.
    mark_delivery.assert_called_once()
    assert not any(
        call.args[0].get("type") == "task_error"
        for call in ws_manager.broadcast_to_task.call_args_list
    )


@pytest.mark.asyncio
async def test_resume_background_restore_broadcast_failure_does_not_affect_restore() -> (
    None
):
    """The corrective post-restore broadcast is best-effort: a failure there
    must not undo the already-committed restore or escape the call."""
    lease = TaskLease(task_id=42, runner_id="runner-a", run_id="run-a")
    settle = MagicMock()
    restore = MagicMock(return_value=True)
    mark_delivery = MagicMock()

    class FakeTracker:
        quota_interrupt_reason = None

        def __init__(self, *, task_id: int, **kwargs: Any) -> None:
            self.complete_tracking = AsyncMock()
            self.stop_periodic_updates = AsyncMock()

        async def start_tracking(self) -> None:
            return None

        async def interrupt_reason_for_quota(self) -> None:
            return None

    agent_service = MagicMock()
    agent_service.resume_execution_by_id = AsyncMock(
        side_effect=CheckpointUnavailableError("checkpoint query failed")
    )

    async def _broadcast(payload: dict, *_args: Any, **_kwargs: Any) -> None:
        if payload.get("type") in {"task_waiting_for_user", "task_paused"}:
            raise RuntimeError("broadcast down")

    ws_manager = MagicMock(broadcast_to_task=AsyncMock(side_effect=_broadcast))

    with _Patches(
        [
            patch(
                "xagent.web.services.task_execution._acquire_resume_task_lease",
                side_effect=_fake_acquire_with_prior_status(
                    lease, TaskStatus.WAITING_FOR_USER
                ),
            ),
            patch(
                "xagent.web.services.task_execution._settle_resumed_task_lease", settle
            ),
            patch(
                "xagent.web.services.task_execution._restore_resumed_task_lease_to_prior_status",
                restore,
            ),
            patch(
                "xagent.web.services.task_execution.mark_user_message_delivery_sync",
                mark_delivery,
            ),
            patch(
                "xagent.web.services.task_execution.run_task_lease_heartbeat",
                new=AsyncMock(),
            ),
            patch(
                "xagent.web.services.task_execution.stop_task_lease_heartbeat",
                new=AsyncMock(),
            ),
            patch("xagent.web.api.websocket.manager", ws_manager),
            patch(
                "xagent.web.services.task_execution.background_task_manager.promote_resume_task"
            ),
            patch("xagent.web.tracking.task_tracker.TaskTracker", FakeTracker),
        ]
    ):
        # Must not raise even though the corrective broadcast fails.
        await execute_resume_background(
            task_id=42,
            agent_service=agent_service,
            task_owner_user_id=1,
            delivery_turn_id="resume-turn",
            expected_run_id="run-a",
            resolved_execution_scope=None,
        )

    settle.assert_not_called()
    restore.assert_called_once()


@pytest.mark.asyncio
async def test_resume_background_checkpoint_unavailable_from_pool_timeout_keeps_lease() -> (
    None
):
    """When the unavailable read is itself pool exhaustion, the existing
    pool-timeout recovery (retain the lease for TTL reaping) wins over the
    new restore-to-prior path -- ``is_database_pool_timeout`` must see
    through the ``CheckpointUnavailableError`` wrapper via its cause chain."""
    lease = TaskLease(task_id=42, runner_id="runner-a", run_id="run-a")
    settle = MagicMock()
    restore = MagicMock(return_value=True)

    def _raise_wrapped_pool_timeout(*args: Any, **kwargs: Any) -> Any:
        try:
            raise SQLAlchemyTimeoutError("pool exhausted")
        except SQLAlchemyTimeoutError as exc:
            raise CheckpointUnavailableError("checkpoint query failed") from exc

    class FakeTracker:
        quota_interrupt_reason = None

        def __init__(self, *, task_id: int, **kwargs: Any) -> None:
            self.complete_tracking = AsyncMock()
            self.stop_periodic_updates = AsyncMock()

        async def start_tracking(self) -> None:
            return None

        async def interrupt_reason_for_quota(self) -> None:
            return None

    agent_service = MagicMock()
    agent_service.resume_execution_by_id = AsyncMock(
        side_effect=_raise_wrapped_pool_timeout
    )
    ws_manager = MagicMock(broadcast_to_task=AsyncMock())

    with _Patches(
        [
            patch(
                "xagent.web.services.task_execution._acquire_resume_task_lease",
                side_effect=_fake_acquire_with_prior_status(
                    lease, TaskStatus.WAITING_FOR_USER
                ),
            ),
            patch(
                "xagent.web.services.task_execution._settle_resumed_task_lease", settle
            ),
            patch(
                "xagent.web.services.task_execution._restore_resumed_task_lease_to_prior_status",
                restore,
            ),
            patch(
                "xagent.web.services.task_execution.run_task_lease_heartbeat",
                new=AsyncMock(),
            ),
            patch(
                "xagent.web.services.task_execution.stop_task_lease_heartbeat",
                new=AsyncMock(),
            ),
            patch("xagent.web.api.websocket.manager", ws_manager),
            patch(
                "xagent.web.services.task_execution.background_task_manager.promote_resume_task"
            ),
            patch("xagent.web.tracking.task_tracker.TaskTracker", FakeTracker),
        ]
    ):
        await execute_resume_background(
            task_id=42,
            agent_service=agent_service,
            task_owner_user_id=1,
            expected_run_id="run-a",
            resolved_execution_scope=None,
        )

    settle.assert_not_called()
    restore.assert_not_called()
    assert not any(
        call.args[0].get("type") == "task_error"
        for call in ws_manager.broadcast_to_task.call_args_list
    )


@pytest.mark.asyncio
async def test_resume_background_marks_failed_on_checkpoint_corrupt() -> None:
    """Corrupt is a terminal state, not a retryable one -- it must fall
    through to the ordinary FAILED settlement, not the restore path."""
    lease = TaskLease(task_id=42, runner_id="runner-a", run_id="run-a")
    settle = MagicMock(return_value=True)
    restore = MagicMock()

    class FakeTracker:
        quota_interrupt_reason = None

        def __init__(self, *, task_id: int, **kwargs: Any) -> None:
            self.complete_tracking = AsyncMock()
            self.stop_periodic_updates = AsyncMock()

        async def start_tracking(self) -> None:
            return None

        async def interrupt_reason_for_quota(self) -> None:
            return None

    agent_service = MagicMock()
    agent_service.resume_execution_by_id = AsyncMock(
        side_effect=CheckpointCorruptError("all matching rows undecodable")
    )
    ws_manager = MagicMock(broadcast_to_task=AsyncMock())

    with _Patches(
        [
            patch(
                "xagent.web.services.task_execution._acquire_resume_task_lease",
                side_effect=_fake_acquire_with_prior_status(
                    lease, TaskStatus.WAITING_FOR_USER
                ),
            ),
            patch(
                "xagent.web.services.task_execution._settle_resumed_task_lease", settle
            ),
            patch(
                "xagent.web.services.task_execution._restore_resumed_task_lease_to_prior_status",
                restore,
            ),
            patch(
                "xagent.web.services.task_execution.run_task_lease_heartbeat",
                new=AsyncMock(),
            ),
            patch(
                "xagent.web.services.task_execution.stop_task_lease_heartbeat",
                new=AsyncMock(),
            ),
            patch("xagent.web.api.websocket.manager", ws_manager),
            patch(
                "xagent.web.services.task_execution.background_task_manager.promote_resume_task"
            ),
            patch("xagent.web.tracking.task_tracker.TaskTracker", FakeTracker),
        ]
    ):
        await execute_resume_background(
            task_id=42,
            agent_service=agent_service,
            task_owner_user_id=1,
            expected_run_id="run-a",
            resolved_execution_scope=None,
        )

    restore.assert_not_called()
    settle.assert_called_once()
    assert any(
        call.args[0].get("type") == "task_error"
        for call in ws_manager.broadcast_to_task.call_args_list
    )


@pytest.mark.asyncio
async def test_resume_tracker_pool_timeout_skips_result_and_lease_checkouts() -> None:
    """A final-usage timeout is the only checkout attempted during cleanup."""
    lease = TaskLease(task_id=42, runner_id="runner-a", run_id="run-a")
    finalize = MagicMock()
    settle = MagicMock()

    class FakeTracker:
        quota_interrupt_reason = None

        def __init__(self, *, task_id: int, **kwargs: Any) -> None:
            assert task_id == 42
            assert kwargs == {
                "expected_run_id": "run-a",
                "expected_runner_id": "runner-a",
                "expected_attempt_id": None,
            }

        async def start_tracking(self) -> None:
            return None

        async def complete_tracking(self) -> None:
            raise SQLAlchemyTimeoutError("pool exhausted")

        async def interrupt_reason_for_quota(self) -> None:
            return None

    agent_service = MagicMock()
    agent_service.resume_execution_by_id = AsyncMock(
        return_value={"status": "completed", "success": True, "output": "ok"}
    )

    with _Patches(
        [
            patch(
                "xagent.web.services.task_execution._acquire_resume_task_lease",
                return_value=lease,
            ),
            patch(
                "xagent.web.services.task_execution._finalize_resumed_task", finalize
            ),
            patch(
                "xagent.web.services.task_execution._settle_resumed_task_lease", settle
            ),
            patch(
                "xagent.web.services.task_execution.run_task_lease_heartbeat",
                new=AsyncMock(),
            ),
            patch(
                "xagent.web.services.task_execution.stop_task_lease_heartbeat",
                new=AsyncMock(),
            ),
            patch(
                "xagent.web.api.websocket.manager",
                MagicMock(broadcast_to_task=AsyncMock()),
            ),
            patch(
                "xagent.web.services.task_execution.background_task_manager.promote_resume_task"
            ),
            patch("xagent.web.tracking.task_tracker.TaskTracker", FakeTracker),
        ]
    ):
        await execute_resume_background(
            task_id=42,
            agent_service=agent_service,
            task_owner_user_id=1,
            resolved_execution_scope=None,
        )

    finalize.assert_not_called()
    settle.assert_not_called()


@pytest.mark.asyncio
async def test_resume_heartbeat_pool_timeout_skips_result_and_lease_checkouts() -> None:
    """Heartbeat exhaustion retains the lease instead of checking out again."""
    lease = TaskLease(task_id=42, runner_id="runner-a", run_id="run-a")
    finalize = MagicMock()
    settle = MagicMock()
    heartbeat_timeout = SQLAlchemyTimeoutError("heartbeat pool exhausted")

    class FakeTracker:
        quota_interrupt_reason = None

        def __init__(self, *, task_id: int, **kwargs: Any) -> None:
            assert task_id == 42
            assert kwargs == {
                "expected_run_id": "run-a",
                "expected_runner_id": "runner-a",
                "expected_attempt_id": None,
            }

        async def start_tracking(self) -> None:
            return None

        async def complete_tracking(self) -> None:
            return None

        async def interrupt_reason_for_quota(self) -> None:
            return None

    agent_service = MagicMock()
    agent_service.resume_execution_by_id = AsyncMock(
        return_value={"status": "completed", "success": True, "output": "ok"}
    )

    with _Patches(
        [
            patch(
                "xagent.web.services.task_execution._acquire_resume_task_lease",
                return_value=lease,
            ),
            patch(
                "xagent.web.services.task_execution._finalize_resumed_task", finalize
            ),
            patch(
                "xagent.web.services.task_execution._settle_resumed_task_lease", settle
            ),
            patch(
                "xagent.web.services.task_execution.run_task_lease_heartbeat",
                new=AsyncMock(),
            ),
            patch(
                "xagent.web.services.task_execution.stop_task_lease_heartbeat",
                new=AsyncMock(
                    return_value=TaskLeaseHeartbeatOutcome(
                        pool_timeout=heartbeat_timeout
                    )
                ),
            ),
            patch(
                "xagent.web.api.websocket.manager",
                MagicMock(broadcast_to_task=AsyncMock()),
            ),
            patch(
                "xagent.web.services.task_execution.background_task_manager.promote_resume_task"
            ),
            patch("xagent.web.tracking.task_tracker.TaskTracker", FakeTracker),
        ]
    ):
        await execute_resume_background(
            task_id=42,
            agent_service=agent_service,
            task_owner_user_id=1,
            resolved_execution_scope=None,
        )

    finalize.assert_not_called()
    settle.assert_not_called()


@pytest.mark.asyncio
async def test_resume_lease_loss_cancels_execution_without_stale_side_effects() -> None:
    lease = TaskLease(task_id=42, runner_id="runner-a", run_id="run-a")
    resume_started = asyncio.Event()
    resume_cancelled = asyncio.Event()
    finalize = MagicMock()
    settle = MagicMock()
    tracker_instances: list[Any] = []

    class FakeTracker:
        quota_interrupt_reason = None

        def __init__(self, *, task_id: int, **kwargs: Any) -> None:
            assert task_id == 42
            assert kwargs == {
                "expected_run_id": "run-a",
                "expected_runner_id": "runner-a",
                "expected_attempt_id": None,
            }
            self.complete_tracking = AsyncMock()
            self.stop_periodic_updates = AsyncMock()
            tracker_instances.append(self)

        async def start_tracking(self) -> None:
            return None

        async def interrupt_reason_for_quota(self) -> None:
            return None

    async def resume(_task_id: str, *, metadata: dict[str, Any]) -> dict[str, Any]:
        assert metadata == {"task_source": None, "run_id": "run-a"}
        resume_started.set()
        try:
            await asyncio.Event().wait()
        finally:
            resume_cancelled.set()

    async def lose_lease(
        _lease: TaskLease,
        _stop_event: asyncio.Event,
    ) -> TaskLeaseHeartbeatOutcome:
        await resume_started.wait()
        return TaskLeaseHeartbeatOutcome(lease_lost=True)

    agent_service = MagicMock()
    agent_service.resume_execution_by_id = AsyncMock(side_effect=resume)
    ws_manager = MagicMock(broadcast_to_task=AsyncMock())

    with _Patches(
        [
            patch(
                "xagent.web.services.task_execution._acquire_resume_task_lease",
                return_value=lease,
            ),
            patch(
                "xagent.web.services.task_execution.run_task_lease_heartbeat",
                new=lose_lease,
            ),
            patch(
                "xagent.web.services.task_execution._finalize_resumed_task", finalize
            ),
            patch(
                "xagent.web.services.task_execution._settle_resumed_task_lease",
                settle,
            ),
            patch("xagent.web.api.websocket.manager", ws_manager),
            patch(
                "xagent.web.services.task_execution.background_task_manager.promote_resume_task"
            ),
            patch(
                "xagent.web.services.task_execution.background_task_manager.cleanup_task"
            ),
            patch("xagent.web.tracking.task_tracker.TaskTracker", FakeTracker),
        ]
    ):
        await asyncio.wait_for(
            execute_resume_background(
                task_id=42,
                agent_service=agent_service,
                task_owner_user_id=1,
                resolved_execution_scope=None,
            ),
            timeout=1,
        )

    assert resume_cancelled.is_set()
    agent_service.resume_execution_by_id.assert_awaited_once_with(
        "42", metadata={"task_source": None, "run_id": "run-a"}
    )
    finalize.assert_not_called()
    settle.assert_not_called()
    assert [
        call.args[0]["type"] for call in ws_manager.broadcast_to_task.call_args_list
    ] == ["task_resumed"]
    assert tracker_instances
    tracker_instances[0].complete_tracking.assert_not_awaited()
    tracker_instances[0].stop_periodic_updates.assert_awaited_once()


@pytest.mark.asyncio
async def test_resume_error_delivery_pool_timeout_skips_lease_checkout() -> None:
    """A timeout while recording failure must not trigger lease settlement."""
    lease = TaskLease(task_id=42, runner_id="runner-a", run_id="run-a")
    settle = MagicMock()
    snapshot = AsyncMock()
    mark_delivery = MagicMock(
        side_effect=SQLAlchemyTimeoutError("delivery pool exhausted")
    )

    class FakeTracker:
        quota_interrupt_reason = None

        def __init__(self, *, task_id: int, **kwargs: Any) -> None:
            assert task_id == 42
            assert kwargs == {
                "expected_run_id": "run-a",
                "expected_runner_id": "runner-a",
                "expected_attempt_id": None,
            }

        async def start_tracking(self) -> None:
            return None

        async def complete_tracking(self) -> None:
            return None

        async def interrupt_reason_for_quota(self) -> None:
            return None

    agent_service = MagicMock()
    agent_service.resume_execution_by_id = AsyncMock(
        side_effect=RuntimeError("resume failed")
    )

    with _Patches(
        [
            patch(
                "xagent.web.services.task_execution._acquire_resume_task_lease",
                return_value=lease,
            ),
            patch(
                "xagent.web.services.task_execution._settle_resumed_task_lease",
                settle,
            ),
            patch(
                "xagent.web.services.task_execution.mark_user_message_delivery_sync",
                mark_delivery,
            ),
            patch(
                "xagent.web.api.websocket.task_execution_controller.snapshot",
                new=snapshot,
            ),
            patch(
                "xagent.web.services.task_execution.run_task_lease_heartbeat",
                new=AsyncMock(),
            ),
            patch(
                "xagent.web.services.task_execution.stop_task_lease_heartbeat",
                new=AsyncMock(),
            ),
            patch(
                "xagent.web.api.websocket.manager",
                MagicMock(broadcast_to_task=AsyncMock()),
            ),
            patch(
                "xagent.web.services.task_execution.background_task_manager.promote_resume_task"
            ),
            patch("xagent.web.tracking.task_tracker.TaskTracker", FakeTracker),
        ]
    ):
        await execute_resume_background(
            task_id=42,
            agent_service=agent_service,
            task_owner_user_id=1,
            delivery_turn_id="resume-turn",
            expected_run_id="run-a",
            resolved_execution_scope=None,
        )

    mark_delivery.assert_called_once()
    snapshot.assert_not_awaited()
    settle.assert_not_called()


@pytest.mark.asyncio
async def test_resume_cancellation_drains_all_final_cleanup() -> None:
    """Caller cancellation must not skip heartbeat stop or exact settlement."""
    lease = TaskLease(task_id=42, runner_id="runner-a", run_id="run-a")
    completion_started = asyncio.Event()
    allow_completion = asyncio.Event()
    events: list[str] = []

    class FakeTracker:
        quota_interrupt_reason = None

        def __init__(self, *, task_id: int, **kwargs: Any) -> None:
            assert task_id == 42
            assert kwargs == {
                "expected_run_id": "run-a",
                "expected_runner_id": "runner-a",
                "expected_attempt_id": None,
            }

        async def start_tracking(self) -> None:
            return None

        async def complete_tracking(self) -> None:
            completion_started.set()
            try:
                await allow_completion.wait()
            except asyncio.CancelledError:
                # Match TaskTracker's contract: drain its inner worker before
                # returning the caller's cancellation.
                await allow_completion.wait()
                events.append("usage")
                raise
            events.append("usage")

        async def interrupt_reason_for_quota(self) -> None:
            return None

    async def stop_heartbeat(*_args: Any) -> None:
        events.append("heartbeat")

    def settle(*_args: Any, **_kwargs: Any) -> bool:
        events.append("lease")
        return True

    agent_service = MagicMock()
    agent_service.resume_execution_by_id = AsyncMock(
        side_effect=RuntimeError("run failed")
    )

    with _Patches(
        [
            patch(
                "xagent.web.services.task_execution._acquire_resume_task_lease",
                return_value=lease,
            ),
            patch(
                "xagent.web.services.task_execution._settle_resumed_task_lease",
                side_effect=settle,
            ),
            patch(
                "xagent.web.services.task_execution.run_task_lease_heartbeat",
                new=AsyncMock(),
            ),
            patch(
                "xagent.web.services.task_execution.stop_task_lease_heartbeat",
                side_effect=stop_heartbeat,
            ),
            patch(
                "xagent.web.api.websocket.manager",
                MagicMock(broadcast_to_task=AsyncMock()),
            ),
            patch(
                "xagent.web.services.task_execution.background_task_manager.promote_resume_task"
            ),
            patch("xagent.web.tracking.task_tracker.TaskTracker", FakeTracker),
        ]
    ):
        execution = asyncio.create_task(
            execute_resume_background(
                task_id=42,
                agent_service=agent_service,
                task_owner_user_id=1,
                resolved_execution_scope=None,
            )
        )
        await asyncio.wait_for(completion_started.wait(), timeout=1)
        execution.cancel()
        await asyncio.sleep(0)
        allow_completion.set()

        with pytest.raises(asyncio.CancelledError):
            await asyncio.wait_for(execution, timeout=1)

    assert events == ["usage", "heartbeat", "lease"]