Skip to content

Enterprise agent platform for building, deploying, and operating AI teammates.

186 KB
Raw Blame
"""Tests for the turn-lifecycle API in ``task_orchestrator``.

Covers:

  - ``TaskTurnPayload`` dual-message channel
  - ``TurnKind`` + ``force_fresh`` orthogonal kind/flag
  - ``begin_turn`` atomic claim + persist + bg schedule
  - ``finish_turn`` symmetric terminal-field writer + lease ownership guard
  - ``_schedule_bg`` lease lifecycle wrapper

Tests use SQLite in-memory + direct ORM, mocking only the bits that
require an actual agent runtime (``execute_task_background``).
"""

from __future__ import annotations

import asyncio
import json
import logging
from concurrent.futures import ThreadPoolExecutor
from contextlib import ExitStack, contextmanager
from datetime import datetime, timedelta, timezone
from threading import Barrier, Event, get_ident
from unittest.mock import AsyncMock, MagicMock, patch

import pytest
import pytest_asyncio
from sqlalchemy import create_engine, text
from sqlalchemy.exc import TimeoutError as SQLAlchemyTimeoutError
from sqlalchemy.orm import Session, sessionmaker
from sqlalchemy.pool import QueuePool

from tests.shared.async_waits import DB_PROGRESS_TIMEOUT
from tests.shared.db_teardown import drop_all_tables
from tests.web.pool_contention_shared import (
    CONTENTION_POOL_TIMEOUT,
    EXHAUSTION_POOL_TIMEOUT,
    GUARD_TIMEOUT,
    LOOP_LIVENESS_TICKS,
    assert_pool_checkout_off_loop,
    gated_pool_checkout,
    wait_for_ticks,
)
from xagent.core.agent.checkpoint import CHECKPOINT_TYPE
from xagent.core.tools.adapters.vibe.config import RequiredMCPUnavailableError
from xagent.core.tools.adapters.vibe.connector_runtime import (
    ConnectorRef,
    ConnectorRuntimeError,
)
from xagent.web.models import database as database_module
from xagent.web.models.agent import Agent
from xagent.web.models.chat_message import TaskChatMessage
from xagent.web.models.database import Base, get_db, get_engine, init_db
from xagent.web.models.task import Task, TaskStatus, TraceEvent
from xagent.web.models.task_execution_event import TaskExecutionEvent
from xagent.web.models.trigger import (
    AgentTrigger,
    TriggerRun,
    TriggerRunStatus,
    TriggerType,
)
from xagent.web.models.user import User
from xagent.web.models.user_channel import UserChannel
from xagent.web.models.workforce import Workforce, WorkforceRun
from xagent.web.services import task_orchestrator as task_orchestrator_module
from xagent.web.services.assistant_history_safety import (
    ASSISTANT_RESPONSE_MESSAGE_TYPE,
    CLIENT_SAFE_FAILURE_MESSAGE_TYPE,
    TASK_FAILURE_MESSAGE_TYPE,
)
from xagent.web.services.chat_history_service import (
    DELIVERY_COMPLETED,
    DELIVERY_DISPATCHED,
    DELIVERY_FAILED,
    DELIVERY_PENDING,
    claim_user_message_delivery_no_commit,
    inspect_user_message_delivery,
    mark_user_message_delivery,
    persist_assistant_message,
)
from xagent.web.services.client_error_messages import (
    CLIENT_SAFE_AUTO_MODEL_UNAVAILABLE,
    CLIENT_SAFE_TASK_FAILURE,
)
from xagent.web.services.connector_runtime import (
    get_ephemeral_runtime_values,
    pop_ephemeral_runtime_values,
    store_ephemeral_runtime_values,
)
from xagent.web.services.external_task_cancel import (
    EXTERNAL_TASK_SOURCE,
    EXTERNAL_TURN_INTERRUPTED_MESSAGE,
)
from xagent.web.services.llm_utils import AutoModelUnavailableError
from xagent.web.services.mcp_runtime import (
    MCPBuiltinOAuthActorPolicy,
)
from xagent.web.services.task_execution_controller import task_execution_controller
from xagent.web.services.task_lease_service import (
    TaskLease,
    TaskLeaseHeartbeatOutcome,
    TaskLeaseRefreshState,
    acquire_task_lease_isolated,
    get_runner_id,
)
from xagent.web.services.task_orchestrator import (
    TaskTurnCommitOutcomeUnknown,
    TaskTurnError,
    TaskTurnNotFoundError,
    TaskTurnOrchestrator,
    TaskTurnPayload,
    TurnKind,
    _begin_turn_atomic_sync,
    _ClaimedTurn,
    _reconcile_finalized_turn_delivery,
    _schedule_bg,
    finish_turn,
    settle_task_lease_isolated,
)
from xagent.web.services.task_runtime import (
    MCP_RUNTIME_AUTHORIZATION_POLICY_REQUIRED_KEY,
)

# ---------------------------------------------------------------------------
# Fixtures
# ---------------------------------------------------------------------------


@pytest.fixture()
def db_session(tmp_path):
    init_db(db_url=f"sqlite:///{tmp_path / 'orchestrator.db'}")
    db = next(get_db())
    try:
        yield db
    finally:
        db.close()
        drop_all_tables(get_engine())


@pytest.fixture()
def queue_pool_runtime_db_factory(tmp_path):
    """Build one-slot QueuePool engines with a caller-chosen checkout timeout."""
    engines = []

    def _make(*, pool_timeout: float = EXHAUSTION_POOL_TIMEOUT):
        db_path = tmp_path / f"orchestrator-queue-pool-{len(engines)}.db"
        engine = create_engine(
            f"sqlite:///{db_path}",
            connect_args={"check_same_thread": False},
            poolclass=QueuePool,
            pool_size=1,
            max_overflow=0,
            pool_timeout=pool_timeout,
        )
        engines.append(engine)
        Base.metadata.create_all(bind=engine)
        SessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine)
        return engine, SessionLocal

    try:
        yield _make
    finally:
        for engine in engines:
            engine.dispose()


@pytest.fixture()
def queue_pool_runtime_db(queue_pool_runtime_db_factory):
    """A real one-slot QueuePool used to exercise checkout exhaustion."""
    return queue_pool_runtime_db_factory()


def _create_user(db) -> User:
    user = User(username="orch-user", password_hash="hash", is_admin=False)
    db.add(user)
    db.commit()
    db.refresh(user)
    return user


def _create_task(
    db,
    user_id: int,
    *,
    status: TaskStatus = TaskStatus.PENDING,
    input_: str | None = None,
    output: str | None = None,
    error_message: str | None = None,
) -> Task:
    task = Task(
        user_id=user_id,
        title="Orchestrator test",
        description="test",
        status=status,
        execution_mode="auto",
        input=input_,
        output=output,
        error_message=error_message,
        source="sdk",
    )
    db.add(task)
    db.commit()
    db.refresh(task)
    return task


def _store_runtime_secret_for_turn(turn_id: str) -> None:
    store_ephemeral_runtime_values(
        turn_id,
        {
            ConnectorRef("mcp", 1): {
                "secrets": {"authorization": "Bearer cleanup-token"}
            }
        },
    )


@pytest.mark.parametrize(
    ("status", "retain"),
    [
        (TaskStatus.WAITING_FOR_USER, True),
        (TaskStatus.PAUSED, True),
        (TaskStatus.RUNNING, True),
        (TaskStatus.PENDING, False),
        (TaskStatus.COMPLETED, False),
        (TaskStatus.FAILED, False),
    ],
)
def test_local_runtime_inputs_follow_run_lifecycle(
    db_session, monkeypatch, status, retain
) -> None:
    from xagent.web.services import connector_runtime as runtime

    monkeypatch.setattr(runtime, "_EPHEMERAL_RUN_VALUES", {})
    user = _create_user(db_session)
    task = _create_task(db_session, int(user.id), status=status)
    task.run_id = "accepted-run"
    db_session.commit()
    turn_id = "lifecycle-inputs"
    _store_runtime_secret_for_turn(turn_id)
    runtime.bind_ephemeral_runtime_values_to_run(
        task_id=int(task.id), run_id=task.run_id, user_id=int(user.id), turn_id=turn_id
    )
    pop_ephemeral_runtime_values(turn_id)

    runtime.clean_ephemeral_runtime_values_for_run(
        task_id=int(task.id), run_id=task.run_id
    )

    assert (runtime._get_ephemeral_run_values(task) is not None) is retain
    # A replacement run must not see the old inputs, even before its cleanup.
    task.run_id = "replacement-run"
    db_session.commit()
    assert runtime._get_ephemeral_run_values(task) is None
    runtime.clean_ephemeral_runtime_values_for_run(
        task_id=int(task.id), run_id="accepted-run"
    )
    assert runtime._EPHEMERAL_RUN_VALUES == {}


def test_local_runtime_inputs_keep_owner_and_original_expiry(
    db_session, monkeypatch
) -> None:
    from xagent.web.services import connector_runtime as runtime

    monkeypatch.setattr(runtime, "_EPHEMERAL_RUN_VALUES", {})
    monkeypatch.setattr(runtime, "monotonic", lambda: 100.0)
    monkeypatch.setenv("XAGENT_TASK_RUNTIME_SECRETS_TTL_SECONDS", "10")
    user = _create_user(db_session)
    task = _create_task(db_session, int(user.id), status=TaskStatus.WAITING_FOR_USER)
    task.run_id = "accepted-run"
    db_session.commit()
    turn_id = "expiry-inputs"
    _store_runtime_secret_for_turn(turn_id)
    runtime.bind_ephemeral_runtime_values_to_run(
        task_id=int(task.id), run_id=task.run_id, user_id=int(user.id), turn_id=turn_id
    )
    pop_ephemeral_runtime_values(turn_id)
    assert runtime._get_ephemeral_run_values(task) is not None
    task.user_id = int(user.id) + 1
    assert runtime._get_ephemeral_run_values(task) is None
    task.user_id = int(user.id)
    monkeypatch.setattr(runtime, "monotonic", lambda: 109.0)
    runtime.bind_ephemeral_runtime_values_to_run(
        task_id=int(task.id), run_id=task.run_id, user_id=int(user.id), turn_id="reply"
    )
    assert runtime._get_ephemeral_run_values(task) is not None
    monkeypatch.setattr(runtime, "monotonic", lambda: 110.0)
    assert runtime._get_ephemeral_run_values(task) is None
    assert runtime._EPHEMERAL_RUN_VALUES == {}


def test_local_runtime_input_bind_purges_previous_run(monkeypatch) -> None:
    from xagent.web.services import connector_runtime as runtime

    monkeypatch.setattr(runtime, "_EPHEMERAL_RUN_VALUES", {})
    for run_id in ("old-run", "new-run"):
        _store_runtime_secret_for_turn(run_id)
        runtime.bind_ephemeral_runtime_values_to_run(
            task_id=1, run_id=run_id, user_id=2, turn_id=run_id
        )
        pop_ephemeral_runtime_values(run_id)

    assert set(runtime._EPHEMERAL_RUN_VALUES) == {(1, "new-run")}


def test_local_runtime_cleanup_sweeps_other_expired_entries(monkeypatch) -> None:
    from xagent.web.services import connector_runtime as runtime

    monkeypatch.setattr(runtime, "_EPHEMERAL_RUN_VALUES", {})
    monkeypatch.setenv("XAGENT_TASK_RUNTIME_SECRETS_TTL_SECONDS", "10")
    for task_id, now in ((1, 100.0), (2, 105.0)):
        monkeypatch.setattr(runtime, "monotonic", lambda: now)
        _store_runtime_secret_for_turn("sweep-inputs")
        runtime.bind_ephemeral_runtime_values_to_run(
            task_id=task_id, run_id="run", user_id=3, turn_id="sweep-inputs"
        )
        pop_ephemeral_runtime_values("sweep-inputs")

    monkeypatch.setattr(runtime, "monotonic", lambda: 110.0)
    # Even cleanup for a task with no retained inputs sweeps other expired
    # records, including entries whose task was deleted while waiting.
    runtime.clean_ephemeral_runtime_values_for_run(task_id=99, run_id="other")
    assert set(runtime._EPHEMERAL_RUN_VALUES) == {(2, "run")}


@pytest.mark.asyncio
async def test_waiting_reply_refreshes_cached_runner_with_required_runtime_inputs(
    db_session, monkeypatch, tmp_path
) -> None:
    from tests.core.agent.test_execution_adapter import (
        FakeLLM,
        FakeTool,
        TracerCheckpointStore,
    )
    from xagent.core.agent.service import AgentService
    from xagent.web.models.custom_api import CustomApi, UserCustomApi
    from xagent.web.services import connector_runtime
    from xagent.web.services.agent_service_manager import AgentServiceManager
    from xagent.web.services.task_execution import background_task_manager
    from xagent.web.tools.config import WebToolConfig

    monkeypatch.setenv("XAGENT_SHARED_TASK_EXECUTION_ENABLED", "false")
    monkeypatch.setattr(connector_runtime, "_EPHEMERAL_RUN_VALUES", {})
    user = _create_user(db_session)
    task = _create_task(db_session, int(user.id))
    task_id, user_id = int(task.id), int(user.id)
    api = CustomApi(
        name="runtime-input-test",
        url="https://example.com/old",
        method="GET",
        runtime_input_schema={
            "secrets": {"token": {"type": "string", "required": True}},
            "auth_selector": {"account": {"type": "string", "required": True}},
        },
    )
    db_session.add(api)
    db_session.flush()
    db_session.add(
        UserCustomApi(
            user_id=user_id, custom_api_id=api.id, is_owner=True, is_active=True
        )
    )
    task.connector_runtime_selected_refs = [
        {"connector_type": "custom_api", "connector_id": int(api.id)}
    ]
    db_session.commit()
    api_id = int(api.id)
    payload = TaskTurnPayload("Read the selected account")
    accepted_inputs = {
        "secrets": {"token": "test-only-token"},
        "auth_selector": {"account": "account-a"},
    }
    store_ephemeral_runtime_values(
        payload.turn_id, {ConnectorRef("custom_api", api_id): accepted_inputs}
    )
    claimed = _begin_turn_atomic_sync(
        task_id, user_id, payload=payload, kind=TurnKind.CREATE
    )
    cfg = WebToolConfig(
        db=None,
        request=None,
        db_factory=database_module.get_session_local(),
        user_id=user_id,
        task_id=str(task_id),
        connector_runtime_turn_id=payload.turn_id,
        workspace_base_dir=str(tmp_path),
    )
    built_tools = []
    built_configs = []

    async def build_tools(config):
        await config.prepare_factory_runtime()
        built_configs.append(config.get_custom_api_configs())
        tool = FakeTool()
        built_tools.append(tool)
        config.handoff_factory_runtime()
        return [tool]

    monkeypatch.setattr(
        "xagent.core.tools.adapters.vibe.factory.ToolFactory.create_all_tools",
        build_tools,
    )
    llm = FakeLLM(
        [
            {
                "tool_calls": [
                    {
                        "id": "ask",
                        "name": "ask_user_question",
                        "args": {"message": "Which period?"},
                    }
                ]
            }
        ]
    )
    service = AgentService(
        name="resume-input-test",
        id=str(task_id),
        llm=llm,
        pattern="react",
        tool_config=cfg,
        enable_workspace=False,
        memory_enabled=False,
        skills_enabled=False,
    )
    service._execution_adapter = service._build_execution_adapter()
    service._execution_adapter.config.tracer = TracerCheckpointStore()
    manager = AgentServiceManager()
    manager._agents[task_id] = service
    manager._agent_owner_ids[task_id] = user_id
    manager._agent_scope_fingerprints[task_id] = None
    results = []

    async def execute(**kwargs):
        if not results:
            result = await service.execute_task(
                "Read account data", task_id=str(task_id)
            )
        else:
            result = await service.resume_execution_by_id(str(task_id))
        results.append(result)
        with database_module.get_session_local()() as db:
            row = db.get(Task, task_id)
            row.status = TaskStatus(result["status"])
            db.commit()

    with (
        patch("xagent.web.services.task_execution.execute_task_background", execute),
        patch(
            "xagent.web.services.task_orchestrator.load_task_setup_snapshot_sync",
            return_value=MagicMock(),
        ),
        patch.object(background_task_manager, "register_task"),
    ):
        await _schedule_bg(
            task_id=task_id,
            task_owner_user_id=user_id,
            task_source="sdk",
            run_id=claimed.run_id,
            task_lease=claimed.task_lease,
            payload=payload,
            force_fresh=False,
            context=None,
        )
        assert results[0]["status"] == "waiting_for_user"
        assert get_ephemeral_runtime_values(payload.turn_id) is None
        db_session.expire_all()
        api = db_session.get(CustomApi, api_id)
        api.url = "https://example.com/new"
        db_session.commit()

        reply = TaskTurnPayload("Last month", turn_id="reply-runtime-input-test")
        cached = await manager.get_agent_for_task(
            task_id,
            task_owner_user_id=user_id,
            connector_runtime_turn_id=reply.turn_id,
            resolved_execution_scope=None,
        )
        assert cached is service
        await cached.post_user_message(
            str(task_id),
            reply.transcript_message,
            turn_id=reply.turn_id,
            request_interrupt=False,
        )
        llm.responses.extend(
            [{"tool_calls": [{"id": "read", "name": "noop", "args": {}}]}, "done"]
        )
        await _schedule_bg(
            task_id=task_id,
            task_owner_user_id=user_id,
            task_source="sdk",
            run_id=claimed.run_id,
            payload=reply,
            force_fresh=False,
            context=None,
        )

    assert results[1]["status"] == "completed"
    assert len(built_tools) == 2
    assert built_tools[0].calls == []
    assert built_tools[1].calls == [{}]
    assert built_configs[0][0]["url"] == "https://example.com/old"
    assert built_configs[1][0]["url"] == "https://example.com/new"
    for config in built_configs:
        assert config[0]["connector_runtime"]["secrets"] == accepted_inputs["secrets"]
        assert (
            config[0]["connector_runtime"]["auth_selector"]
            == accepted_inputs["auth_selector"]
        )
    assert (task_id, claimed.run_id) not in connector_runtime._EPHEMERAL_RUN_VALUES
    cfg.close()


def test_channel_and_web_claims_are_cross_process_exclusive(db_session) -> None:
    user = _create_user(db_session)
    task = _create_task(db_session, int(user.id))
    task_id = int(task.id)
    barrier = Barrier(2)

    def claim_channel() -> str | None:
        barrier.wait()
        lease = acquire_task_lease_isolated(
            task_id,
            runner_id="channel-runner",
            new_run=True,
        )
        return lease.run_id if lease is not None else None

    def claim_web() -> str | None:
        barrier.wait()
        try:
            claimed = _begin_turn_atomic_sync(
                task_id,
                int(user.id),
                payload=TaskTurnPayload("web turn"),
                kind=TurnKind.CREATE,
            )
        except TaskTurnError:
            return None
        return claimed.run_id

    with ThreadPoolExecutor(max_workers=2) as executor:
        channel_result = executor.submit(claim_channel)
        web_result = executor.submit(claim_web)
        winners = [
            run_id
            for run_id in (channel_result.result(), web_result.result())
            if run_id is not None
        ]

    assert len(winners) == 1
    db_session.expire_all()
    stored = db_session.query(Task).filter(Task.id == task_id).one()
    assert stored.status == TaskStatus.RUNNING
    assert stored.run_id == winners[0]


@pytest.fixture()
def mock_schedule_bg():
    """Stub the bg coroutine spawn so begin_turn tests don't actually run
    an agent. Opt-in: tests that drive ``_schedule_bg`` directly skip
    this fixture and patch deeper layers themselves.

    Uses ``AsyncMock()`` without an explicit ``return_value`` —
    instantiating ``asyncio.Future()`` at fixture-setup time needs a
    running event loop, which pytest-asyncio doesn't provide during
    fixture collection in CI. The default ``AsyncMock`` return is a
    plain ``MagicMock``, which begin_turn ignores anyway.
    """
    with patch(
        "xagent.web.services.task_orchestrator._schedule_bg",
        new=MagicMock(),
    ) as mocked:
        yield mocked


@pytest.fixture(autouse=True)
def _clear_bg_manager():
    """Reset the global bg manager between tests so _refuse_if_bg_inflight
    sees a clean slate."""
    from xagent.web.services.task_execution import background_task_manager

    background_task_manager.running_tasks.clear()
    yield
    background_task_manager.running_tasks.clear()


# ---------------------------------------------------------------------------
# TaskTurnPayload
# ---------------------------------------------------------------------------


def test_payload_for_agent_falls_back_to_transcript() -> None:
    p = TaskTurnPayload(transcript_message="hi")
    assert p.for_agent == "hi"


def test_payload_uses_execution_when_provided() -> None:
    p = TaskTurnPayload(
        transcript_message="summarize this",
        execution_message="summarize this\n\n[file context]",
    )
    assert p.for_agent == "summarize this\n\n[file context]"


def test_execution_context_carries_path_stripped_turn_attachments() -> None:
    files = [{"file_id": "image-id", "name": "screen.png", "type": "image/png"}]

    context = task_orchestrator_module._execution_context_with_turn_id(
        {"existing": True},
        "turn-1",
        files=files,
    )

    assert context == {
        "existing": True,
        "turn_id": "turn-1",
        "files": files,
    }
    assert context["files"] is not files


def test_execution_context_preserves_existing_turn_attachments() -> None:
    existing_files = [
        {"file_id": "existing", "name": "existing.png", "type": "image/png"}
    ]

    context = task_orchestrator_module._execution_context_with_turn_id(
        {"files": existing_files},
        "turn-2",
        files=[{"file_id": "new", "name": "new.png", "type": "image/png"}],
    )

    assert context["files"] is existing_files


# ---------------------------------------------------------------------------
# begin_turn — happy paths
# ---------------------------------------------------------------------------


@pytest.mark.asyncio
async def test_begin_turn_create_clears_no_terminal_fields_when_pending(
    db_session,
    mock_schedule_bg,
) -> None:
    user = _create_user(db_session)
    task = _create_task(db_session, user.id, status=TaskStatus.PENDING)
    payload = TaskTurnPayload("first turn")

    started = await TaskTurnOrchestrator.begin_turn(
        task_id=int(task.id),
        payload=payload,
        task_owner_user_id=int(user.id),
        kind=TurnKind.CREATE,
        force_fresh=False,
    )

    db_session.refresh(task)
    assert task.status == TaskStatus.RUNNING
    assert task.input == "first turn"
    assert task.output is None
    assert task.error_message is None
    assert task.runner_id == get_runner_id()
    assert task.lease_expires_at is not None
    assert task.run_id == started.run_id
    assert task.state_version == started.state_version == 1
    assert task.control_state == started.control_state == "running"
    delivery = (
        db_session.query(TaskChatMessage)
        .filter(
            TaskChatMessage.turn_id == payload.turn_id,
            TaskChatMessage.role == "user",
        )
        .one()
    )
    assert delivery.delivery_status == DELIVERY_DISPATCHED


@pytest.mark.asyncio
async def test_begin_turn_projects_workforce_running_in_claim_transaction(
    db_session,
    mock_schedule_bg,
) -> None:
    user = _create_user(db_session)
    manager = Agent(user_id=user.id, name="claim projection manager")
    db_session.add(manager)
    db_session.flush()
    workforce = Workforce(
        owner_user_id=user.id,
        scope_type="user",
        scope_id=str(user.id),
        name="claim projection workforce",
        manager_agent_id=manager.id,
        status="active",
    )
    db_session.add(workforce)
    db_session.flush()
    task = _create_task(db_session, user.id, status=TaskStatus.PENDING)
    run = WorkforceRun(
        workforce_id=workforce.id,
        task_id=task.id,
        user_id=user.id,
        status="failed",
        snapshot={"version": 1},
        completed_at=datetime.now(timezone.utc),
    )
    db_session.add(run)
    db_session.flush()
    task.agent_config = {"workforce_run_id": int(run.id)}
    db_session.commit()

    await TaskTurnOrchestrator.begin_turn(
        task_id=int(task.id),
        task_owner_user_id=int(user.id),
        payload=TaskTurnPayload("retry"),
        kind=TurnKind.CREATE,
    )

    db_session.expire_all()
    persisted_run = db_session.get(WorkforceRun, int(run.id))
    assert persisted_run.status == "running"
    assert persisted_run.completed_at is None


@pytest.mark.asyncio
async def test_begin_turn_claim_replaces_stale_lease_and_checkpoint_pointer(
    db_session,
    mock_schedule_bg,
) -> None:
    """Append claim must reset stale lease columns so the bg runner can
    acquire after a crashed worker left runner_id behind."""
    from xagent.web.services.task_lease_service import utc_now

    user = _create_user(db_session)
    task = _create_task(
        db_session,
        user.id,
        status=TaskStatus.COMPLETED,
        input_="done",
        output="answer",
    )
    task.runner_id = "dead-runner-from-crash"
    task.lease_attempt_id = "test-attempt"
    task.lease_expires_at = utc_now() + timedelta(hours=1)
    task.last_checkpoint_event_id = "previous-run-checkpoint"
    stale_checkpoint = TraceEvent(
        task_id=task.id,
        event_id="previous-run-checkpoint",
        event_type="system_update_general",
        timestamp=utc_now(),
        data={"checkpoint_type": CHECKPOINT_TYPE, "snapshot": {"type": "checkpoint"}},
    )
    db_session.add(stale_checkpoint)
    db_session.flush()
    task.last_checkpoint_trace_event_id = stale_checkpoint.id
    db_session.commit()

    await TaskTurnOrchestrator.begin_turn(
        task_id=int(task.id),
        payload=TaskTurnPayload("follow up"),
        task_owner_user_id=int(user.id),
        kind=TurnKind.APPEND,
        force_fresh=False,
    )

    db_session.refresh(task)
    assert task.status == TaskStatus.RUNNING
    assert task.runner_id == get_runner_id()
    assert task.lease_expires_at is not None
    assert task.last_checkpoint_event_id is None
    # Both pointer columns clear together on a turn claim -- see
    # lease_checkpoint_trace_event_id_case's docstring in task_lease_service
    # for why a single column left behind would desync the recovery CAS
    # fence.
    assert task.last_checkpoint_trace_event_id is None


@pytest.mark.asyncio
async def test_begin_turn_append_clears_stale_output_and_error(
    db_session,
    mock_schedule_bg,
) -> None:
    """Latest-turn snapshot invariant: appending a new turn must reset
    output / error_message from the previous turn so GET returns a
    coherent latest-turn snapshot.
    """
    user = _create_user(db_session)
    task = _create_task(
        db_session,
        user.id,
        status=TaskStatus.COMPLETED,
        input_="first question",
        output="first answer",
        error_message=None,
    )

    await TaskTurnOrchestrator.begin_turn(
        task_id=int(task.id),
        payload=TaskTurnPayload("second question"),
        task_owner_user_id=int(user.id),
        kind=TurnKind.APPEND,
        force_fresh=False,
    )

    db_session.refresh(task)
    assert task.status == TaskStatus.RUNNING
    assert task.input == "second question"
    assert task.output is None, "stale first-turn output must be cleared"
    assert task.error_message is None


@pytest.mark.asyncio
async def test_begin_turn_append_clears_stale_error_message(
    db_session,
    mock_schedule_bg,
) -> None:
    """Latest-turn snapshot invariant (FAILED side): appending after a
    failed turn must also clear the prior turn's error_message."""
    user = _create_user(db_session)
    task = _create_task(
        db_session,
        user.id,
        status=TaskStatus.FAILED,
        input_="first",
        output=None,
        error_message="first turn blew up",
    )

    await TaskTurnOrchestrator.begin_turn(
        task_id=int(task.id),
        payload=TaskTurnPayload("second"),
        task_owner_user_id=int(user.id),
        kind=TurnKind.APPEND,
        force_fresh=False,
    )

    db_session.refresh(task)
    assert task.status == TaskStatus.RUNNING
    assert task.input == "second"
    assert task.error_message is None
    assert task.output is None


@pytest.mark.asyncio
@pytest.mark.parametrize(
    "followup",
    [
        "The connection is restored. Continue the requested report.",
        "The connection is restored. Use site B instead; keep the other filters.",
    ],
)
async def test_append_snapshot_retains_original_request_after_blocked_answer(
    db_session,
    mock_schedule_bg,
    followup: str,
) -> None:
    """A fresh turn needs the user's request, not just the blocked answer."""
    from xagent.web.services.task_setup_snapshot import load_task_setup_snapshot_sync

    user = _create_user(db_session)
    task = _create_task(db_session, user.id)
    original = (
        "Read site A records for 2026-09-24, exclude cancellations, and total cost."
    )
    await TaskTurnOrchestrator.begin_turn(
        task_id=int(task.id),
        task_owner_user_id=int(user.id),
        payload=TaskTurnPayload(original),
        kind=TurnKind.CREATE,
    )
    db_session.refresh(task)
    # A blocked final_answer ends the run but does not repeat the request's
    # parameters. None can be recovered from this reply or Task.description.
    blocked = "Access expired. Reconnect the service to continue."
    persist_assistant_message(
        db_session,
        int(task.id),
        int(user.id),
        blocked,
        message_type=ASSISTANT_RESPONSE_MESSAGE_TYPE,
    )
    task.status = TaskStatus.COMPLETED
    db_session.commit()
    assert finish_turn(db_session, int(task.id))

    payload = TaskTurnPayload(followup)
    await TaskTurnOrchestrator.begin_turn(
        task_id=int(task.id),
        task_owner_user_id=int(user.id),
        payload=payload,
        kind=TurnKind.APPEND,
    )
    scheduled = mock_schedule_bg.call_args.kwargs
    current_row = (
        db_session.query(TaskChatMessage)
        .filter(TaskChatMessage.turn_id == payload.turn_id)
        .one()
    )
    assert scheduled["before_message_id"] == int(current_row.id)
    snapshot = load_task_setup_snapshot_sync(
        int(task.id),
        int(user.id),
        before_message_id=scheduled["before_message_id"],
    )
    assert snapshot is not None
    assert list(snapshot.conversation_history) == [
        {"role": "user", "content": original},
        {"role": "assistant", "content": blocked},
    ]
    # The current request is sent separately, not duplicated in history or
    # replaced with the original task. This also preserves user corrections.
    assert scheduled["payload"].for_agent == followup
    assert [
        row.content
        for row in db_session.query(TaskChatMessage)
        .filter(TaskChatMessage.task_id == task.id, TaskChatMessage.role == "user")
        .order_by(TaskChatMessage.id)
        .all()
    ] == [original, followup]


@pytest.mark.asyncio
async def test_begin_turn_append_accepts_paused_task_as_new_turn(
    db_session,
    mock_schedule_bg,
) -> None:
    """A message sent after pause starts the next turn, not a checkpoint resume."""
    user = _create_user(db_session)
    task = _create_task(
        db_session,
        user.id,
        status=TaskStatus.PAUSED,
        input_="previous request",
        output="stale partial output",
        error_message="stale pause detail",
    )

    payload = TaskTurnPayload("new request after pause")
    await TaskTurnOrchestrator.begin_turn(
        task_id=int(task.id),
        payload=payload,
        task_owner_user_id=int(user.id),
        kind=TurnKind.APPEND,
        force_fresh=False,
    )

    db_session.refresh(task)
    assert task.status == TaskStatus.RUNNING
    assert task.input == "new request after pause"
    assert task.output is None
    assert task.error_message is None

    persisted = (
        db_session.query(TaskChatMessage)
        .filter(TaskChatMessage.task_id == int(task.id), TaskChatMessage.role == "user")
        .one()
    )
    assert persisted.content == "new request after pause"
    assert persisted.turn_id == payload.turn_id

    mock_schedule_bg.assert_called_once()


@pytest.mark.asyncio
async def test_begin_turn_passes_force_fresh_through_to_schedule_bg(
    db_session,
    mock_schedule_bg,
) -> None:
    """Dual-channel payload + force_fresh forwarding: begin_turn forwards
    the full ``TaskTurnPayload`` and ``force_fresh`` flag to
    ``_schedule_bg`` so the execution side receives both message
    channels and the right reconstruct-state mode."""
    user = _create_user(db_session)
    task = _create_task(db_session, user.id, status=TaskStatus.COMPLETED)

    payload = TaskTurnPayload(
        transcript_message="show me",
        execution_message="show me\n\n[file: foo.pdf]",
    )
    await TaskTurnOrchestrator.begin_turn(
        task_id=int(task.id),
        payload=payload,
        task_owner_user_id=int(user.id),
        kind=TurnKind.APPEND,
        force_fresh=True,
    )

    mock_schedule_bg.assert_called_once()
    kwargs = mock_schedule_bg.call_args.kwargs
    assert kwargs["payload"] is payload
    assert kwargs["force_fresh"] is True

    persisted = (
        db_session.query(TaskChatMessage)
        .filter(TaskChatMessage.task_id == int(task.id), TaskChatMessage.role == "user")
        .one()
    )
    assert persisted.turn_id == payload.turn_id


def test_begin_turn_reconciles_a_commit_acknowledgement_failure(
    db_session,
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    user = _create_user(db_session)
    task = _create_task(db_session, user.id, status=TaskStatus.PENDING)
    payload = TaskTurnPayload("acknowledged after disconnect")
    original_commit = Session.commit
    commit_raised = False

    def acknowledge_then_disconnect(session: Session) -> None:
        nonlocal commit_raised
        original_commit(session)
        if session is not db_session and not commit_raised:
            commit_raised = True
            raise ConnectionError("commit acknowledgement lost")

    monkeypatch.setattr(Session, "commit", acknowledge_then_disconnect)

    claimed = _begin_turn_atomic_sync(
        int(task.id),
        int(user.id),
        payload=payload,
        kind=TurnKind.CREATE,
    )

    assert commit_raised is True
    assert claimed.run_id
    db_session.expire_all()
    stored = db_session.query(TaskChatMessage).filter_by(turn_id=payload.turn_id).one()
    assert stored.content == "acknowledged after disconnect"


def test_begin_turn_retires_failed_session_before_reconciliation(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    events: list[str] = []
    commit_error = ConnectionError("commit acknowledgement lost")
    lease = MagicMock(runner_id="runner", run_id="run")
    claimed = _ClaimedTurn(
        task_lease=lease,
        status=TaskStatus.RUNNING,
        updated_at=None,
        before_message_id=None,
        task_source="sdk",
        run_id="run",
    )

    class FailedSession:
        def flush(self) -> None:
            events.append("flush")

        def commit(self) -> None:
            events.append("commit")
            raise commit_error

        def rollback(self) -> None:
            events.append("rollback")

        def close(self) -> None:
            events.append("close")

    monkeypatch.setattr(database_module, "_SessionLocal", lambda: FailedSession())
    monkeypatch.setattr(
        task_orchestrator_module,
        "_claim_turn_no_commit",
        MagicMock(return_value=claimed),
    )

    def reconcile(**_kwargs) -> bool:
        events.append("reconcile")
        assert events.index("close") < events.index("reconcile")
        return True

    monkeypatch.setattr(
        task_orchestrator_module,
        "_reconcile_claimed_turn_after_commit_ack_failure",
        reconcile,
    )

    result = _begin_turn_atomic_sync(
        123,
        456,
        payload=TaskTurnPayload("accepted", turn_id="retire-before-reconcile"),
        kind=TurnKind.CREATE,
    )

    assert result is claimed
    assert events[:4] == ["flush", "commit", "close", "reconcile"]


def test_commit_reconciliation_accepts_a_late_visible_graph(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    lease = MagicMock(runner_id="runner", run_id="run")
    claimed = _ClaimedTurn(
        task_lease=lease,
        status=TaskStatus.RUNNING,
        updated_at=None,
        before_message_id=None,
        task_source="sdk",
        run_id="run",
    )
    payload = TaskTurnPayload("accepted", turn_id="late-visible-turn")
    sessions = [MagicMock(), MagicMock(), MagicMock()]
    sessions[0].query.return_value.filter.return_value.first.return_value = None
    sessions[1].query.return_value.filter.return_value.first.return_value = None
    task_query = MagicMock()
    task_query.filter.return_value.first.return_value = MagicMock()
    message_query = MagicMock()
    message_query.filter.return_value.first.return_value = MagicMock()
    third_queries = [task_query, message_query]
    sessions[2].query.side_effect = third_queries
    created_sessions: list[MagicMock] = []

    def session_factory() -> MagicMock:
        session = sessions[len(created_sessions)]
        created_sessions.append(session)
        return session

    monkeypatch.setattr(database_module, "_SessionLocal", session_factory)
    monkeypatch.setattr(task_orchestrator_module.time, "sleep", MagicMock())

    assert (
        task_orchestrator_module._reconcile_claimed_turn_after_commit_ack_failure(
            task_id=123,
            task_owner_user_id=456,
            payload=payload,
            claimed=claimed,
        )
        is True
    )
    assert len(created_sessions) == 3
    assert all(session.close.called for session in created_sessions)


def test_successful_commit_is_not_rejected_when_session_close_fails(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    events: list[str] = []
    claimed = _ClaimedTurn(
        task_lease=MagicMock(runner_id="runner", run_id="run"),
        status=TaskStatus.RUNNING,
        updated_at=None,
        before_message_id=None,
        task_source="sdk",
        run_id="run",
    )

    class CloseFailingSession:
        def flush(self) -> None:
            events.append("flush")

        def commit(self) -> None:
            events.append("commit")

        def rollback(self) -> None:
            events.append("rollback")

        def close(self) -> None:
            events.append("close")
            raise RuntimeError("close failed")

        def invalidate(self) -> None:
            events.append("invalidate")

    monkeypatch.setattr(database_module, "_SessionLocal", lambda: CloseFailingSession())
    monkeypatch.setattr(
        task_orchestrator_module,
        "_claim_turn_no_commit",
        MagicMock(return_value=claimed),
    )
    invalidate_cache = MagicMock()
    monkeypatch.setattr(
        task_orchestrator_module,
        "invalidate_task_cache_best_effort",
        invalidate_cache,
    )

    result = _begin_turn_atomic_sync(
        123,
        456,
        payload=TaskTurnPayload("accepted", turn_id="close-failure-turn"),
        kind=TurnKind.CREATE,
    )

    assert result is claimed
    assert events == ["flush", "commit", "close", "invalidate"]
    invalidate_cache.assert_called_once_with(123)


def test_reconciliation_read_failure_remains_commit_outcome_unknown(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    commit_error = ConnectionError("commit acknowledgement lost")
    claimed = _ClaimedTurn(
        task_lease=MagicMock(runner_id="runner", run_id="run"),
        status=TaskStatus.RUNNING,
        updated_at=None,
        before_message_id=None,
        task_source="sdk",
        run_id="run",
    )
    factory_calls = 0

    class FailedCommitSession:
        def flush(self) -> None:
            return None

        def commit(self) -> None:
            raise commit_error

        def rollback(self) -> None:
            return None

        def close(self) -> None:
            return None

    def session_factory():
        nonlocal factory_calls
        factory_calls += 1
        if factory_calls == 1:
            return FailedCommitSession()
        raise RuntimeError("reconciliation database unavailable")

    monkeypatch.setattr(database_module, "_SessionLocal", session_factory)
    monkeypatch.setattr(
        task_orchestrator_module,
        "_claim_turn_no_commit",
        MagicMock(return_value=claimed),
    )
    monkeypatch.setattr(task_orchestrator_module.time, "sleep", MagicMock())

    with pytest.raises(TaskTurnCommitOutcomeUnknown) as exc_info:
        _begin_turn_atomic_sync(
            123,
            456,
            payload=TaskTurnPayload(
                "accepted",
                turn_id="reconciliation-read-failure",
            ),
            kind=TurnKind.CREATE,
        )

    assert exc_info.value.__cause__ is commit_error
    assert factory_calls == 4


@pytest.mark.asyncio
async def test_begin_turn_actor_is_audit_only_not_runtime(
    db_session,
    mock_schedule_bg,
    caplog,
) -> None:
    """``actor_user_id`` records who initiated the turn (an admin acting on
    another user's task) for audit/logging only. It must never reach the
    claim or the bg schedule -- those run as the OWNER -- and it must appear
    in the audit log line."""
    owner = _create_user(db_session)
    task = _create_task(db_session, owner.id, status=TaskStatus.COMPLETED)
    actor_id = int(owner.id) + 999  # a different principal (e.g. an admin)

    with caplog.at_level(logging.INFO, logger="xagent.web.services.task_orchestrator"):
        await TaskTurnOrchestrator.begin_turn(
            task_id=int(task.id),
            payload=TaskTurnPayload("follow-up"),
            task_owner_user_id=int(owner.id),
            actor_user_id=actor_id,
            kind=TurnKind.APPEND,
        )

    # The bg schedule runs as the owner; the actor never leaks into it.
    kwargs = mock_schedule_bg.call_args.kwargs
    assert kwargs["task_owner_user_id"] == int(owner.id)
    assert actor_id not in kwargs.values()

    # The persisted user message is attributed to the owner, not the actor.
    persisted = (
        db_session.query(TaskChatMessage)
        .filter(TaskChatMessage.task_id == int(task.id), TaskChatMessage.role == "user")
        .one()
    )
    assert persisted.user_id == int(owner.id)

    # The actor is captured in the audit log.
    assert "turn started" in caplog.text
    assert f"owner={int(owner.id)}" in caplog.text
    assert f"actor={actor_id}" in caplog.text


# ---------------------------------------------------------------------------
# begin_turn — failure modes
# ---------------------------------------------------------------------------


@pytest.mark.asyncio
async def test_begin_turn_rejects_create_with_force_fresh(
    db_session,
    mock_schedule_bg,
) -> None:
    """Invalid kind + flag combo: CREATE + force_fresh has no meaning."""
    user = _create_user(db_session)
    task = _create_task(db_session, user.id, status=TaskStatus.PENDING)

    with pytest.raises(ValueError, match="force_fresh has no meaning"):
        await TaskTurnOrchestrator.begin_turn(
            task_id=int(task.id),
            payload=TaskTurnPayload("x"),
            task_owner_user_id=int(user.id),
            kind=TurnKind.CREATE,
            force_fresh=True,
        )


@pytest.mark.asyncio
async def test_begin_turn_rejects_task_not_owned_by_user(
    db_session,
    mock_schedule_bg,
) -> None:
    """Ownership is folded into the atomic claim predicate. A ``user_id``
    that does not own the task → ``TaskTurnNotFoundError`` (404), NOT
    ``TaskTurnError`` (409), and no row is mutated. Passing a *different*
    user id (not ``task.user_id``) proves the predicate actually guards."""
    user = _create_user(db_session)
    task = _create_task(db_session, user.id, status=TaskStatus.PENDING)

    with pytest.raises(TaskTurnNotFoundError):
        await TaskTurnOrchestrator.begin_turn(
            task_id=int(task.id),
            task_owner_user_id=int(user.id) + 9999,
            payload=TaskTurnPayload("x"),
            kind=TurnKind.CREATE,
        )

    db_session.refresh(task)
    assert task.status == TaskStatus.PENDING, "rejected claim must not mutate"
    mock_schedule_bg.assert_not_called()


@pytest.mark.asyncio
async def test_begin_turn_marks_failed_when_schedule_raises(
    db_session,
) -> None:
    """Post-commit invariant: once the claim commits (RUNNING) but
    ``_schedule_bg`` raises, the task must be forced FAILED so it is never
    left RUNNING with no bg worker (zombie)."""
    user = _create_user(db_session)
    task = _create_task(db_session, user.id, status=TaskStatus.PENDING)
    payload = TaskTurnPayload("x")

    with patch(
        "xagent.web.services.task_orchestrator._schedule_bg",
        new=MagicMock(side_effect=RuntimeError("schedule boom")),
    ):
        with pytest.raises(RuntimeError, match="schedule boom"):
            await TaskTurnOrchestrator.begin_turn(
                task_id=int(task.id),
                task_owner_user_id=int(user.id),
                payload=payload,
                kind=TurnKind.CREATE,
            )

    db_session.refresh(task)
    assert task.status == TaskStatus.FAILED
    delivery = (
        db_session.query(TaskChatMessage)
        .filter(
            TaskChatMessage.turn_id == payload.turn_id,
            TaskChatMessage.role == "user",
        )
        .one()
    )
    assert delivery.delivery_status == DELIVERY_FAILED


def test_schedule_failure_compensation_does_not_fail_foreign_live_lease(
    db_session,
) -> None:
    """Compensation only owns the still-unleased claim it just committed.

    Another scheduler may acquire the same run before the failed scheduler's
    compensation worker reaches the database.  Matching the run alone must not
    let that stale compensation overwrite the live runner's execution.
    """
    user = _create_user(db_session)
    manager = Agent(user_id=user.id, name="compensation manager")
    db_session.add(manager)
    db_session.flush()
    workforce = Workforce(
        owner_user_id=user.id,
        scope_type="user",
        scope_id=str(user.id),
        name="compensation workforce",
        manager_agent_id=manager.id,
        status="active",
    )
    db_session.add(workforce)
    db_session.flush()
    task = _create_task(db_session, user.id, status=TaskStatus.RUNNING)
    run = WorkforceRun(
        workforce_id=workforce.id,
        task_id=task.id,
        user_id=user.id,
        status="running",
        snapshot={"version": 1},
    )
    db_session.add(run)
    db_session.flush()
    task.agent_config = {"workforce_run_id": int(run.id)}
    task.run_id = "claimed-run"
    task.runner_id = "replacement-runner"
    task.lease_attempt_id = "test-attempt"
    task.lease_expires_at = datetime.now(timezone.utc) + timedelta(minutes=5)
    task.error_message = "replacement still running"
    db_session.commit()

    assert (
        settle_task_lease_isolated(
            TaskLease(
                task_id=int(task.id),
                runner_id="stale-runner",
                run_id="claimed-run",
                attempt_id="test-attempt",
            ),
            error_message="stale schedule failure",
        )
        is False
    )

    db_session.expire_all()
    persisted = db_session.query(Task).filter(Task.id == task.id).one()
    assert persisted.status == TaskStatus.RUNNING
    assert persisted.run_id == "claimed-run"
    assert persisted.runner_id == "replacement-runner"
    assert persisted.error_message == "replacement still running"
    persisted_run = (
        db_session.query(WorkforceRun).filter(WorkforceRun.id == run.id).one()
    )
    assert persisted_run.status == "running"


def test_error_settlement_releases_terminal_lease_without_reporting_failure(
    db_session,
) -> None:
    """A post-commit presentation error must not rewrite or announce COMPLETED."""

    user = _create_user(db_session)
    task = _create_task(db_session, user.id, status=TaskStatus.COMPLETED)
    task.run_id = "completed-run"
    task.runner_id = "completed-runner"
    task.lease_attempt_id = "test-attempt"
    task.lease_expires_at = datetime.now(timezone.utc) + timedelta(minutes=5)
    db_session.add(
        TaskChatMessage(
            task_id=int(task.id),
            user_id=int(user.id),
            role="assistant",
            content="durable result",
            message_type="assistant_message",
        )
    )
    db_session.commit()

    error_committed = settle_task_lease_isolated(
        TaskLease(
            task_id=int(task.id),
            runner_id="completed-runner",
            run_id="completed-run",
            attempt_id="test-attempt",
        ),
        error_message="completion broadcast failed",
    )

    assert error_committed is False
    db_session.expire_all()
    persisted = db_session.query(Task).filter(Task.id == task.id).one()
    assert persisted.status == TaskStatus.COMPLETED
    assert persisted.output == "durable result"
    assert persisted.error_message is None
    assert persisted.runner_id is None
    assert persisted.lease_expires_at is None


@pytest.mark.parametrize("storage_version", [1, 2])
def test_error_settlement_persists_client_safe_history(
    db_session, storage_version: int
) -> None:
    secret = "orchestrator-provider-secret"
    turn_id = f"settlement-turn-v{storage_version}"
    user = _create_user(db_session)
    task = _create_task(db_session, user.id, status=TaskStatus.RUNNING)
    task.conversation_storage_version = storage_version
    db_session.commit()
    task_id = int(task.id)
    lease = acquire_task_lease_isolated(task_id)
    assert lease is not None

    assert (
        settle_task_lease_isolated(
            lease,
            error_message=f"setup/run failed: {secret}",
            turn_id=turn_id,
        )
        is True
    )

    db_session.expire_all()
    persisted = db_session.get(Task, task_id)
    assert persisted is not None
    assert persisted.error_message == f"setup/run failed: {secret}"
    messages = (
        db_session.query(TaskChatMessage)
        .filter(
            TaskChatMessage.task_id == task_id,
            TaskChatMessage.role == "assistant",
        )
        .all()
    )
    assert len(messages) == 1
    assert messages[0].content == "Task execution failed."
    assert messages[0].turn_id == turn_id
    if storage_version == 2:
        facts = (
            db_session.query(TaskExecutionEvent)
            .filter(TaskExecutionEvent.task_id == task_id)
            .order_by(TaskExecutionEvent.sequence)
            .all()
        )
        assert [fact.kind for fact in facts] == [
            "assistant_message",
            "execution_settled",
        ]
        assert [fact.turn_id for fact in facts] == [turn_id, turn_id]
    assert secret not in messages[0].content


@pytest.mark.asyncio
async def test_begin_turn_dispatch_timeout_does_not_reject_scheduled_turn(
    db_session,
    mock_schedule_bg,
    caplog,
) -> None:
    """A post-schedule delivery write is best-effort under pool exhaustion.

    The claim is already committed and the background task is already
    scheduled.  Reporting the checkout timeout to the API would tell the
    caller that a turn which is actually running was rejected.
    """
    user = _create_user(db_session)
    task = _create_task(db_session, user.id, status=TaskStatus.PENDING)
    payload = TaskTurnPayload("start once")

    caplog.set_level(logging.ERROR, logger="xagent.web.services.task_orchestrator")
    with patch(
        "xagent.web.services.task_orchestrator.mark_user_message_delivery_sync",
        side_effect=SQLAlchemyTimeoutError("delivery pool exhausted"),
    ) as mark_delivery:
        started = await TaskTurnOrchestrator.begin_turn(
            task_id=int(task.id),
            task_owner_user_id=int(user.id),
            payload=payload,
            kind=TurnKind.CREATE,
        )

    assert started.task_id == int(task.id)
    mock_schedule_bg.assert_called_once()
    mark_delivery.assert_called_once_with(
        int(task.id), payload.turn_id, DELIVERY_DISPATCHED
    )
    db_session.expire_all()
    stored_task = db_session.query(Task).filter(Task.id == task.id).one()
    assert stored_task.status == TaskStatus.RUNNING
    delivery = (
        db_session.query(TaskChatMessage)
        .filter(TaskChatMessage.turn_id == payload.turn_id)
        .one()
    )
    assert delivery.delivery_status == DELIVERY_PENDING
    assert "component=turn-delivery database pool checkout timed out" in caplog.text


@pytest.mark.asyncio
async def test_begin_turn_dispatch_projection_failure_keeps_scheduled_turn(
    db_session: Session,
    mock_schedule_bg: MagicMock,
    caplog: pytest.LogCaptureFixture,
) -> None:
    """A failed delivery projection cannot undo a successful schedule.

    The durable task claim and background handle are already committed facts.
    A non-pool failure while projecting ``dispatched`` must leave the turn
    running, return the original handle, and avoid compensation or reschedule.
    """
    user = _create_user(db_session)
    task = _create_task(db_session, int(user.id), status=TaskStatus.PENDING)
    payload = TaskTurnPayload("start exactly once")

    caplog.set_level(logging.ERROR, logger="xagent.web.services.task_orchestrator")
    with (
        patch(
            "xagent.web.services.task_orchestrator.mark_user_message_delivery_sync",
            side_effect=RuntimeError("delivery projection failed"),
        ) as mark_delivery,
        patch(
            "xagent.web.services.task_orchestrator.settle_task_lease_isolated"
        ) as compensate,
    ):
        started = await TaskTurnOrchestrator.begin_turn(
            task_id=int(task.id),
            task_owner_user_id=int(user.id),
            payload=payload,
            kind=TurnKind.CREATE,
        )

    assert started.background_task is mock_schedule_bg.return_value
    mock_schedule_bg.assert_called_once()
    mark_delivery.assert_called_once_with(
        int(task.id), payload.turn_id, DELIVERY_DISPATCHED
    )
    compensate.assert_not_called()
    db_session.expire_all()
    stored_task = db_session.query(Task).filter(Task.id == task.id).one()
    assert stored_task.status == TaskStatus.RUNNING
    delivery = (
        db_session.query(TaskChatMessage)
        .filter(TaskChatMessage.turn_id == payload.turn_id)
        .one()
    )
    assert delivery.delivery_status == DELIVERY_PENDING
    assert "component=turn-delivery projection failed after scheduling" in caplog.text


@pytest.mark.asyncio
async def test_schedule_failure_pool_timeout_does_not_checkout_delivery_again(
    db_session,
    caplog,
) -> None:
    """A failed terminal write is the last checkout after schedule failure."""
    user = _create_user(db_session)
    task = _create_task(db_session, user.id, status=TaskStatus.PENDING)
    payload = TaskTurnPayload("cannot schedule")
    mark_delivery = MagicMock()
    settle = MagicMock(side_effect=SQLAlchemyTimeoutError("terminal pool exhausted"))

    caplog.set_level(logging.ERROR, logger="xagent.web.services.task_orchestrator")
    with (
        patch(
            "xagent.web.services.task_orchestrator._schedule_bg",
            side_effect=RuntimeError("schedule boom"),
        ),
        patch(
            "xagent.web.services.task_orchestrator.settle_task_lease_isolated",
            settle,
        ),
        patch(
            "xagent.web.services.task_orchestrator.mark_user_message_delivery_sync",
            mark_delivery,
        ),
        pytest.raises(RuntimeError, match="schedule boom"),
    ):
        await TaskTurnOrchestrator.begin_turn(
            task_id=int(task.id),
            task_owner_user_id=int(user.id),
            payload=payload,
            kind=TurnKind.CREATE,
        )

    settle.assert_called_once()
    mark_delivery.assert_not_called()
    assert (
        "component=turn-schedule-terminal database pool checkout timed out"
        in caplog.text
    )


@pytest.mark.asyncio
async def test_begin_turn_schedules_even_when_caller_cancelled(db_session) -> None:
    """Cancellation safety: if begin_turn's caller is cancelled while the
    off-loop claim is in flight (which commits RUNNING in a worker thread),
    the owned claim+schedule task must settle before cancellation propagates,
    so a committed RUNNING task is never left with no scheduled worker."""
    import threading

    user = _create_user(db_session)
    task = _create_task(db_session, user.id, status=TaskStatus.PENDING)

    claim_started = threading.Event()
    release_claim = threading.Event()

    def slow_claim(
        task_id,
        task_owner_user_id,
        *,
        payload,
        kind,
        context=None,
        force_fresh=False,
        actor_user_id=None,
    ):
        claim_started.set()
        assert release_claim.wait(timeout=GUARD_TIMEOUT)
        return _ClaimedTurn(
            task_lease=TaskLease(
                task_id=task_id,
                runner_id=get_runner_id(),
                run_id="slow-claim-run",
                attempt_id="test-attempt",
            ),
            status=TaskStatus.RUNNING,
            updated_at=datetime.now(timezone.utc),
            before_message_id=1,
            task_source="sdk",
            run_id="slow-claim-run",
        )

    sched = MagicMock(return_value=MagicMock())
    with (
        patch(
            "xagent.web.services.task_orchestrator._begin_turn_atomic_sync",
            new=slow_claim,
        ),
        patch(
            "xagent.web.services.task_orchestrator._schedule_bg",
            new=sched,
        ),
    ):
        t = asyncio.create_task(
            TaskTurnOrchestrator.begin_turn(
                task_id=int(task.id),
                task_owner_user_id=int(user.id),
                payload=TaskTurnPayload("x"),
                kind=TurnKind.CREATE,
            )
        )
        try:
            assert await asyncio.to_thread(claim_started.wait, GUARD_TIMEOUT)
            t.cancel()
        finally:
            release_claim.set()
            await asyncio.wait_for(
                asyncio.gather(t, return_exceptions=True), timeout=GUARD_TIMEOUT
            )
        with pytest.raises(asyncio.CancelledError):
            t.result()

    sched.assert_called_once()  # scheduled despite the cancellation


@pytest.mark.asyncio
async def test_repeated_cancellation_keeps_turn_command_gate_until_claim_settles(
    db_session,
) -> None:
    """A second cancellation cannot release the per-task command gate while
    the owned claim is still committing in its worker thread."""
    import threading

    user = _create_user(db_session)
    task = _create_task(db_session, user.id, status=TaskStatus.PENDING)
    claim_started = threading.Event()
    release_claim = threading.Event()
    contender_entered = asyncio.Event()

    def blocked_claim(
        task_id,
        task_owner_user_id,
        *,
        payload,
        kind,
        context=None,
        force_fresh=False,
        actor_user_id=None,
    ):
        claim_started.set()
        assert release_claim.wait(timeout=30), "claim was never released"
        return _ClaimedTurn(
            task_lease=TaskLease(
                task_id=task_id,
                runner_id=get_runner_id(),
                run_id="blocked-claim-run",
                attempt_id="test-attempt",
            ),
            status=TaskStatus.RUNNING,
            updated_at=datetime.now(timezone.utc),
            before_message_id=1,
            task_source="sdk",
            run_id="blocked-claim-run",
        )

    async def contender() -> None:
        async with task_execution_controller.command(int(task.id)):
            contender_entered.set()

    sched = MagicMock(return_value=MagicMock())
    with (
        patch(
            "xagent.web.services.task_orchestrator._begin_turn_atomic_sync",
            new=blocked_claim,
        ),
        patch("xagent.web.services.task_orchestrator._schedule_bg", new=sched),
    ):
        turn = asyncio.create_task(
            TaskTurnOrchestrator.begin_turn(
                task_id=int(task.id),
                task_owner_user_id=int(user.id),
                payload=TaskTurnPayload("x"),
                kind=TurnKind.CREATE,
            )
        )
        contender_task = None
        try:
            assert await asyncio.to_thread(claim_started.wait, 30)

            turn.cancel()
            await asyncio.sleep(0.01)
            turn.cancel()
            contender_task = asyncio.create_task(contender())
            await asyncio.sleep(0.05)

            assert not turn.done()
            assert not contender_entered.is_set()

            release_claim.set()
            with pytest.raises(asyncio.CancelledError):
                await turn
            await asyncio.wait_for(contender_task, timeout=30)
        finally:
            release_claim.set()
            pending = [turn]
            if contender_task is not None:
                pending.append(contender_task)
            await asyncio.wait_for(
                asyncio.gather(*pending, return_exceptions=True), timeout=30
            )

    assert contender_entered.is_set()
    sched.assert_called_once()


@pytest.mark.asyncio
async def test_schedule_claimed_create_turn_offloads_cache_invalidation(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    """A committed domain claim must not invalidate Redis on the event loop."""
    task_id = 987654
    event_loop_thread = get_ident()
    invalidations: list[tuple[int, int]] = []
    entered = Event()
    release = Event()

    def slow_invalidate(observed_task_id: int) -> None:
        invalidations.append((observed_task_id, get_ident()))
        entered.set()
        assert get_ident() != event_loop_thread
        assert release.wait(timeout=30), "cache invalidation was never released"

    async def fake_schedule(**_kwargs):
        async def done() -> None:
            return None

        return asyncio.create_task(done())

    monkeypatch.setattr(
        task_orchestrator_module,
        "invalidate_task_cache",
        slow_invalidate,
    )
    monkeypatch.setattr(
        task_orchestrator_module,
        "_schedule_committed_turn",
        fake_schedule,
    )
    claimed = _ClaimedTurn(
        task_lease=TaskLease(
            task_id=task_id,
            runner_id=get_runner_id(),
            run_id="committed-run",
            attempt_id="test-attempt",
        ),
        status=TaskStatus.RUNNING,
        updated_at=None,
        before_message_id=None,
        task_source="internal",
        run_id="committed-run",
    )

    startup = asyncio.create_task(
        TaskTurnOrchestrator.schedule_claimed_create_turn(
            task_id=task_id,
            task_owner_user_id=1,
            actor_user_id=1,
            payload=TaskTurnPayload("start"),
            claimed=claimed,
        )
    )
    try:
        assert await asyncio.to_thread(entered.wait, 30)
        assert not startup.done()
    finally:
        release.set()
        started = await asyncio.wait_for(startup, timeout=30)
    await started.background_task

    assert len(invalidations) == 1
    assert invalidations[0][0] == task_id
    assert invalidations[0][1] != event_loop_thread


@pytest.mark.asyncio
async def test_begin_turn_refuses_when_bg_inflight(
    db_session,
    mock_schedule_bg,
) -> None:
    from xagent.web.services.task_execution import background_task_manager

    user = _create_user(db_session)
    task = _create_task(db_session, user.id, status=TaskStatus.COMPLETED)

    # Plant a fake "still-running" entry in the bg manager registry.
    # ``_refuse_if_bg_inflight`` only checks ``.done() is False``, so a
    # MagicMock with that one attribute is enough — we don't need a
    # real asyncio.Task (and creating one would require an extra event
    # loop, which trips up pytest-asyncio's fixture machinery in CI).
    fake_inflight = MagicMock(spec=asyncio.Task)
    fake_inflight.done.return_value = False
    background_task_manager.running_tasks[int(task.id)] = fake_inflight

    try:
        with pytest.raises(TaskTurnError) as excinfo:
            await TaskTurnOrchestrator.begin_turn(
                task_id=int(task.id),
                payload=TaskTurnPayload("x"),
                task_owner_user_id=int(user.id),
                kind=TurnKind.APPEND,
            )
        assert excinfo.value.reason == "bg_inflight"

        # Critical: the DB row must NOT have been mutated
        db_session.refresh(task)
        assert task.status == TaskStatus.COMPLETED  # unchanged
        assert task.input is None  # unchanged
    finally:
        background_task_manager.running_tasks.pop(int(task.id), None)


@pytest.mark.asyncio
async def test_begin_turn_refuses_create_against_terminal_task(
    db_session,
    mock_schedule_bg,
) -> None:
    """kind=CREATE filters status==PENDING; a COMPLETED task must reject."""
    user = _create_user(db_session)
    task = _create_task(db_session, user.id, status=TaskStatus.COMPLETED)

    with pytest.raises(TaskTurnError) as excinfo:
        await TaskTurnOrchestrator.begin_turn(
            task_id=int(task.id),
            payload=TaskTurnPayload("x"),
            task_owner_user_id=int(user.id),
            kind=TurnKind.CREATE,
        )
    assert excinfo.value.reason == "busy"


@pytest.mark.asyncio
async def test_begin_turn_refuses_append_against_pending_task(
    db_session,
    mock_schedule_bg,
) -> None:
    """kind=APPEND filters status IN TERMINAL; a PENDING task must reject."""
    user = _create_user(db_session)
    task = _create_task(db_session, user.id, status=TaskStatus.PENDING)

    with pytest.raises(TaskTurnError) as excinfo:
        await TaskTurnOrchestrator.begin_turn(
            task_id=int(task.id),
            payload=TaskTurnPayload("x"),
            task_owner_user_id=int(user.id),
            kind=TurnKind.APPEND,
        )
    assert excinfo.value.reason == "busy"


# ---------------------------------------------------------------------------
# finish_turn
# ---------------------------------------------------------------------------


def test_finish_turn_completed_writes_output_clears_error(db_session) -> None:
    from xagent.web.models.chat_message import TaskChatMessage

    user = _create_user(db_session)
    task = _create_task(
        db_session,
        user.id,
        status=TaskStatus.COMPLETED,
        error_message="stale",
    )
    msg = TaskChatMessage(
        task_id=task.id,
        user_id=user.id,
        role="assistant",
        content="hello world",
        message_type="assistant_message",
    )
    db_session.add(msg)
    db_session.commit()

    finish_turn(db_session, int(task.id))

    db_session.refresh(task)
    assert task.output == "hello world"
    assert task.error_message is None


def test_finish_turn_failed_writes_error_clears_stale_output(db_session) -> None:
    """Latest-turn snapshot invariant (FAILED side): a FAILED turn
    must clear the prior turn's stale ``output`` so the GET response
    doesn't show ``status='failed' + output='prior answer'``."""
    user = _create_user(db_session)
    task = _create_task(
        db_session,
        user.id,
        status=TaskStatus.FAILED,
        output="prior successful output",
        error_message=None,
    )

    finish_turn(db_session, int(task.id))

    db_session.refresh(task)
    assert task.error_message is not None
    assert "Task execution failed" in task.error_message
    assert task.output is None  # latest-turn snapshot invariant


def test_finish_turn_running_skips_when_other_worker_holds_live_lease(
    db_session,
) -> None:
    """Lease ownership guard: when another worker actively holds the
    lease, finish_turn must leave the row alone and not flip RUNNING
    to FAILED."""
    user = _create_user(db_session)
    task = _create_task(db_session, user.id, status=TaskStatus.RUNNING)
    # Plant a live lease held by a different runner
    task.runner_id = "other-worker"
    task.lease_attempt_id = "test-attempt"
    task.lease_expires_at = datetime.now(timezone.utc) + timedelta(minutes=5)
    task.output = "other worker's in-progress output"
    db_session.commit()

    finish_turn(db_session, int(task.id))

    db_session.refresh(task)
    # No change: status stays RUNNING, output untouched, no error injected
    assert task.status == TaskStatus.RUNNING
    assert task.runner_id == "other-worker"
    assert task.output == "other worker's in-progress output"
    assert task.error_message is None


def test_finish_turn_running_flips_failed_when_no_live_lease(db_session) -> None:
    """RUNNING + no live lease elsewhere → genuine stuck task → flip FAILED."""
    user = _create_user(db_session)
    task = _create_task(
        db_session,
        user.id,
        status=TaskStatus.RUNNING,
        output="stale partial output",
    )
    # No runner_id / lease — task is stuck
    db_session.commit()

    finish_turn(db_session, int(task.id))

    db_session.refresh(task)
    assert task.status == TaskStatus.FAILED
    assert task.error_message is not None
    assert task.output is None  # latest-turn snapshot invariant


def test_finish_turn_running_flips_failed_when_lease_expired(db_session) -> None:
    """RUNNING + lease present but expired → still flip FAILED."""
    user = _create_user(db_session)
    task = _create_task(db_session, user.id, status=TaskStatus.RUNNING)
    task.runner_id = "other-worker"
    task.lease_attempt_id = "test-attempt"
    task.lease_expires_at = datetime.now(timezone.utc) - timedelta(minutes=5)
    db_session.commit()

    finish_turn(db_session, int(task.id))

    db_session.refresh(task)
    assert task.status == TaskStatus.FAILED


def test_finish_turn_running_flips_failed_when_we_own_lease(db_session) -> None:
    """RUNNING + we own the lease ourselves → still our bug to finalize."""
    user = _create_user(db_session)
    task = _create_task(db_session, user.id, status=TaskStatus.RUNNING)
    task.runner_id = get_runner_id()  # our own process
    task.lease_attempt_id = "test-attempt"
    task.lease_expires_at = datetime.now(timezone.utc) + timedelta(minutes=5)
    db_session.commit()

    finish_turn(db_session, int(task.id))

    db_session.refresh(task)
    assert task.status == TaskStatus.FAILED


def test_finish_turn_does_not_touch_a_new_run_owned_by_same_process(
    db_session,
) -> None:
    """The concrete lease, not the process-global runner id, fences cleanup.

    A worker process can finish an old coroutine after the same process has
    already claimed a newer run.  Matching only ``get_runner_id()`` would let
    the stale coroutine fail the new run because both leases share that value.
    """
    user = _create_user(db_session)
    task = _create_task(db_session, user.id, status=TaskStatus.RUNNING)
    task.runner_id = get_runner_id()
    task.lease_attempt_id = "test-attempt"
    task.run_id = "new-run"
    task.lease_expires_at = datetime.now(timezone.utc) + timedelta(minutes=5)
    db_session.commit()

    stale_lease = TaskLease(
        task_id=int(task.id),
        runner_id=get_runner_id(),
        run_id="old-run",
        attempt_id="test-attempt",
    )
    finish_turn(db_session, int(task.id), task_lease=stale_lease)

    db_session.expire_all()
    persisted = db_session.query(Task).filter(Task.id == task.id).one()
    assert persisted.status == TaskStatus.RUNNING
    assert persisted.runner_id == get_runner_id()
    assert persisted.run_id == "new-run"
    assert persisted.error_message is None


def test_finish_turn_cache_invalidation_failure_is_non_fatal_after_release(
    db_session,
) -> None:
    """A committed exact-lease settlement must not be reported as failed."""
    user = _create_user(db_session)
    task = _create_task(db_session, user.id, status=TaskStatus.FAILED)
    task.runner_id = "settlement-runner"
    task.lease_attempt_id = "test-attempt"
    task.run_id = "settlement-run"
    task.error_message = "execution failed"
    db_session.commit()
    lease = TaskLease(
        task_id=int(task.id),
        runner_id="settlement-runner",
        run_id="settlement-run",
        attempt_id="test-attempt",
    )

    with patch(
        "xagent.web.services.task_orchestrator.invalidate_task_cache",
        side_effect=RuntimeError("cache backend unavailable"),
    ):
        assert finish_turn(db_session, int(task.id), task_lease=lease) is True

    db_session.expire_all()
    persisted = db_session.query(Task).filter(Task.id == task.id).one()
    assert persisted.status == TaskStatus.FAILED
    assert persisted.runner_id is None
    assert persisted.lease_expires_at is None


def test_finish_turn_mirrors_failed_task_to_trigger_run(db_session) -> None:
    """The existing terminal-state owner mirrors setup failures to TriggerRun."""
    user = _create_user(db_session)
    agent = Agent(user_id=user.id, name="trigger agent")
    db_session.add(agent)
    db_session.flush()
    trigger = AgentTrigger(
        user_id=user.id,
        agent_id=agent.id,
        type=TriggerType.SCHEDULED.value,
        name="scheduled test",
        config={},
    )
    db_session.add(trigger)
    db_session.flush()
    task = _create_task(
        db_session,
        user.id,
        status=TaskStatus.FAILED,
        error_message="Required MCP servers are unavailable.",
    )
    task.source = "trigger"
    run = TriggerRun(
        trigger_id=trigger.id,
        task_id=task.id,
        status=TriggerRunStatus.RUNNING.value,
        idempotency_key="strict-mcp-setup-failure",
    )
    db_session.add_all([task, run])
    db_session.commit()

    finish_turn(db_session, int(task.id))

    db_session.refresh(run)
    assert run.status == TriggerRunStatus.FAILED.value
    assert run.error_message == "Required MCP servers are unavailable."


# ---------------------------------------------------------------------------
# _schedule_bg lease lifecycle
# ---------------------------------------------------------------------------


@pytest.mark.asyncio
async def test_schedule_bg_cancels_execution_after_heartbeat_loses_lease(
    db_session,
) -> None:
    from xagent.web.services.task_execution import background_task_manager

    user = _create_user(db_session)
    task = _create_task(db_session, user.id, status=TaskStatus.RUNNING)
    task.runner_id = "test-runner"
    task.lease_attempt_id = "test-attempt"
    task.run_id = "run-a"
    db_session.commit()
    lease = TaskLease(int(task.id), "test-runner", "run-a", attempt_id="test-attempt")
    execution_started = asyncio.Event()
    execution_cancelled = asyncio.Event()

    async def heartbeat(*_args, **_kwargs) -> TaskLeaseHeartbeatOutcome:
        await execution_started.wait()
        return TaskLeaseHeartbeatOutcome(lease_lost=True)

    async def execute(**_kwargs) -> None:
        execution_started.set()
        try:
            await asyncio.Event().wait()
        finally:
            execution_cancelled.set()

    with (
        patch(
            "xagent.web.services.task_orchestrator.acquire_task_lease_isolated",
            return_value=lease,
        ),
        patch(
            "xagent.web.services.task_orchestrator.run_task_lease_heartbeat",
            new=heartbeat,
        ),
        patch(
            "xagent.web.services.task_orchestrator.load_task_setup_snapshot_sync",
            return_value=MagicMock(),
        ),
        patch.object(
            task_orchestrator_module,
            "resolve_execution_scope",
            return_value=None,
        ),
        patch(
            "xagent.web.services.task_execution.execute_task_background",
            new=execute,
        ),
        patch(
            "xagent.web.services.task_orchestrator.settle_task_lease_isolated",
        ) as settle,
        patch.object(background_task_manager, "register_task"),
        patch(
            "xagent.web.services.task_orchestrator._get_agent_manager",
            return_value=MagicMock(),
        ),
    ):
        await _schedule_bg(
            task_id=int(task.id),
            task_owner_user_id=int(user.id),
            task_source=task.source,
            run_id="run-a",
            payload=TaskTurnPayload("hello"),
            force_fresh=False,
            context=None,
        )

    assert execution_cancelled.is_set()
    settle.assert_not_called()


@pytest.mark.asyncio
async def test_delayed_preclaimed_scheduler_does_not_resurrect_recovered_task(
    db_session,
) -> None:
    from xagent.web.services.task_execution import background_task_manager
    from xagent.web.services.task_lease_recovery import (
        recover_expired_task_leases_until_cutoff,
    )
    from xagent.web.services.task_lease_service import utc_now

    user = _create_user(db_session)
    task = _create_task(db_session, user.id, status=TaskStatus.PENDING)
    payload = TaskTurnPayload("claimed before scheduler delay")
    claimed = _begin_turn_atomic_sync(
        int(task.id),
        int(user.id),
        payload=payload,
        kind=TurnKind.CREATE,
    )
    db_session.expire_all()
    persisted = db_session.get(Task, int(task.id))
    persisted.lease_expires_at = utc_now() - timedelta(seconds=1)
    db_session.commit()

    assert (
        await recover_expired_task_leases_until_cutoff(
            cutoff=utc_now(),
            batch_size=10,
        )
        == 1
    )

    with (
        patch(
            "xagent.web.services.task_orchestrator.run_task_lease_heartbeat",
            new=AsyncMock(),
        ),
        patch(
            "xagent.web.services.task_execution.execute_task_background",
            new=AsyncMock(),
        ) as execute,
        patch(
            "xagent.web.services.task_orchestrator.settle_task_lease_isolated",
        ) as settle,
        patch.object(background_task_manager, "register_task"),
    ):
        await _schedule_bg(
            task_id=int(task.id),
            task_owner_user_id=int(user.id),
            task_source=task.source,
            run_id=claimed.run_id,
            task_lease=claimed.task_lease,
            payload=payload,
            force_fresh=False,
            context=None,
        )

    execute.assert_not_awaited()
    settle.assert_not_called()
    db_session.expire_all()
    assert db_session.get(Task, int(task.id)).status == TaskStatus.FAILED


@pytest.mark.asyncio
async def test_schedule_bg_skips_finish_turn_when_lease_acquire_fails(
    db_session,
) -> None:
    """Running-elsewhere short-circuit: lease taken by another worker
    → never call execute_task_background or finish_turn; bg coroutine
    returns clean."""
    user = _create_user(db_session)
    task = _create_task(db_session, user.id, status=TaskStatus.RUNNING)

    from xagent.web.services.task_execution import background_task_manager

    payload = TaskTurnPayload("x")
    _store_runtime_secret_for_turn(payload.turn_id)
    assert get_ephemeral_runtime_values(payload.turn_id) is not None

    with (
        patch(
            "xagent.web.services.task_orchestrator.acquire_task_lease_isolated",
            return_value=None,
        ),
        patch(
            "xagent.web.services.task_execution.execute_task_background",
            new=AsyncMock(),
        ) as mock_exec,
        patch(
            "xagent.web.services.task_orchestrator.finish_turn",
        ) as mock_finish,
        patch.object(background_task_manager, "register_task"),
    ):
        # Note: this test does NOT use the mock_schedule_bg fixture
        # because we're testing _schedule_bg itself. The real
        # function runs with the deeper layers patched.
        bg_task = _schedule_bg(
            task_id=int(task.id),
            task_owner_user_id=int(user.id),
            task_source=task.source,
            payload=payload,
            force_fresh=False,
            context=None,
        )
        await bg_task

    mock_exec.assert_not_awaited()
    mock_finish.assert_not_called()
    assert get_ephemeral_runtime_values(payload.turn_id) is None
    assert pop_ephemeral_runtime_values(payload.turn_id) is None


@pytest.mark.asyncio
async def test_schedule_bg_resolves_scope_off_loop_before_execution(
    db_session,
    queue_pool_runtime_db_factory,
) -> None:
    """A contended scope lookup must not block the main event loop."""
    from xagent.web.services.task_execution import background_task_manager

    user = _create_user(db_session)
    task = _create_task(db_session, user.id, status=TaskStatus.RUNNING)
    task_id = int(task.id)
    fake_lease = TaskLease(
        task_id=task_id, runner_id="test-runner", attempt_id="test-attempt"
    )
    # The scope lookup must wait for the slot, never give up on it.
    engine, SessionLocal = queue_pool_runtime_db_factory(
        pool_timeout=CONTENTION_POOL_TIMEOUT
    )
    events: list[str] = []
    resolver_threads: list[int] = []
    loop_thread = get_ident()

    def load_snapshot(*_args, **_kwargs):
        events.append("snapshot")
        return MagicMock()

    def resolve_scope(resolved_task_id):
        assert resolved_task_id == task_id
        resolver_threads.append(get_ident())
        with SessionLocal() as scope_db:
            scope_db.execute(text("SELECT 1")).scalar()
        events.append("scope")
        return None

    async def execute(**kwargs):
        events.append("execute")
        assert "resolved_execution_scope" in kwargs
        assert kwargs["resolved_execution_scope"] is None

    held_connection = engine.connect()
    ticker_stop = asyncio.Event()
    ticks = 0

    async def ticker() -> None:
        nonlocal ticks
        while not ticker_stop.is_set():
            await asyncio.sleep(0.01)
            ticks += 1

    with (
        gated_pool_checkout(engine) as gate,
        patch(
            "xagent.web.services.task_orchestrator.acquire_task_lease_isolated",
            return_value=fake_lease,
        ),
        patch(
            "xagent.web.services.task_orchestrator.run_task_lease_heartbeat",
            new=AsyncMock(),
        ),
        patch(
            "xagent.web.services.task_orchestrator.load_task_setup_snapshot_sync",
            side_effect=load_snapshot,
        ),
        patch.object(
            task_orchestrator_module,
            "resolve_execution_scope",
            side_effect=resolve_scope,
            create=True,
        ),
        patch(
            "xagent.web.services.task_execution.execute_task_background",
            new=execute,
        ),
        patch(
            "xagent.web.services.task_orchestrator.settle_task_lease_isolated",
        ),
        patch.object(background_task_manager, "register_task"),
        patch(
            "xagent.web.services.task_orchestrator._get_agent_manager",
            return_value=MagicMock(),
        ),
    ):
        bg_task = _schedule_bg(
            task_id=task_id,
            task_owner_user_id=int(user.id),
            task_source=task.source,
            payload=TaskTurnPayload("hello"),
            force_fresh=False,
            context=None,
        )
        ticker_task = asyncio.create_task(ticker())
        try:
            await gate.wait_until_contending()
            observed = await wait_for_ticks(lambda: ticks)
            assert observed >= LOOP_LIVENESS_TICKS, (
                "scope QueuePool checkout blocked the event loop"
            )
            assert not bg_task.done(), "execution started before scope resolution"
        finally:
            held_connection.close()
            gate.let_through()
            await asyncio.wait_for(bg_task, timeout=GUARD_TIMEOUT)
            ticker_stop.set()
            await ticker_task

    assert events == ["snapshot", "scope", "execute"]
    assert resolver_threads and resolver_threads[0] != loop_thread


@pytest.mark.parametrize(
    ("failure_point", "expect_flag"),
    [
        ("missing_snapshot", True),
        ("execute_error", True),
        ("success", False),
        ("paused", False),
    ],
)
@pytest.mark.asyncio
async def test_schedule_bg_settles_off_loop_and_registers_failure_flag(
    db_session,
    failure_point,
    expect_flag,
) -> None:
    from xagent.core.agent.interruption import InterruptionReason
    from xagent.web.services.task_execution import background_task_manager

    user = _create_user(db_session)
    task = _create_task(db_session, user.id, status=TaskStatus.RUNNING)
    task_id = int(task.id)
    fake_lease = TaskLease(
        task_id=task_id, runner_id="test-runner", attempt_id="test-attempt"
    )
    loop_thread = get_ident()
    marker_threads: list[int] = []
    snapshot = None if failure_point == "missing_snapshot" else MagicMock()
    execute = AsyncMock(
        side_effect=(
            RuntimeError("simulated execute failure")
            if failure_point in ("execute_error", "paused")
            else None
        )
    )

    def record_failure(*_args, **settle_kwargs) -> bool | None:
        marker_threads.append(get_ident())
        if failure_point == "paused":
            # The settlement reports the handled failure as a pause.
            settle_kwargs["report"].paused_for = InterruptionReason.PERSISTENCE_FAILURE
            return True
        return None

    with (
        patch(
            "xagent.web.services.task_orchestrator.acquire_task_lease_isolated",
            return_value=fake_lease,
        ),
        patch(
            "xagent.web.services.task_orchestrator.run_task_lease_heartbeat",
            new=AsyncMock(),
        ),
        patch(
            "xagent.web.services.task_orchestrator.load_task_setup_snapshot_sync",
            return_value=snapshot,
        ),
        patch.object(
            task_orchestrator_module,
            "resolve_execution_scope",
            return_value=None,
            create=True,
        ),
        patch(
            "xagent.web.services.task_execution.execute_task_background",
            new=execute,
        ),
        patch(
            "xagent.web.services.task_orchestrator.settle_task_lease_isolated",
            side_effect=record_failure,
        ),
        patch.object(background_task_manager, "register_task") as register,
        patch(
            "xagent.web.services.task_orchestrator.publish_settlement_pause",
            new=AsyncMock(),
        ) as publish_pause,
        patch(
            "xagent.web.services.task_orchestrator._get_agent_manager",
            return_value=MagicMock(),
        ),
    ):
        await _schedule_bg(
            task_id=task_id,
            task_owner_user_id=int(user.id),
            task_source=task.source,
            payload=TaskTurnPayload("hello"),
            force_fresh=False,
            context=None,
        )
        (_, bg_task), kwargs = register.call_args
        await bg_task

    assert marker_threads and marker_threads[0] != loop_thread
    # The runner settles every outcome and returns normally; the registration's
    # failure flag is what reports a handled failure to the per-process
    # execution counts, and a pause is not a failure.
    assert bg_task.exception() is None
    assert kwargs["failure_flag"]() is expect_flag
    assert publish_pause.await_count == int(failure_point == "paused")


@pytest.mark.asyncio
async def test_schedule_bg_pool_timeout_defers_settlement_to_lease_recovery(
    queue_pool_runtime_db,
    caplog,
) -> None:
    """One exhausted checkout must not trigger an immediate second checkout."""
    from xagent.web.services.task_execution import background_task_manager

    engine, SessionLocal = queue_pool_runtime_db
    with SessionLocal() as seed_db:
        user = _create_user(seed_db)
        task = _create_task(seed_db, user.id, status=TaskStatus.RUNNING)
        task_id = int(task.id)
        user_id = int(user.id)
        task_source = task.source
        task.runner_id = "test-runner"
        task.lease_attempt_id = "test-attempt"
        task.run_id = "run-a"
        seed_db.commit()

    lease = TaskLease(
        task_id=task_id,
        runner_id="test-runner",
        run_id="run-a",
        attempt_id="test-attempt",
    )

    def load_snapshot_from_contended_pool(*_args, **_kwargs):
        with SessionLocal() as snapshot_db:
            snapshot_db.execute(text("SELECT 1")).scalar()
        return MagicMock()

    held_connection = engine.connect()
    caplog.set_level(
        logging.ERROR,
        logger="xagent.web.services.task_orchestrator",
    )
    try:
        with (
            assert_pool_checkout_off_loop(engine),
            patch(
                "xagent.web.services.task_orchestrator.acquire_task_lease_isolated",
                return_value=lease,
            ),
            patch(
                "xagent.web.services.task_orchestrator.run_task_lease_heartbeat",
                new=AsyncMock(),
            ),
            patch(
                "xagent.web.services.task_orchestrator.stop_task_lease_heartbeat",
                new=AsyncMock(),
            ) as mock_stop_heartbeat,
            patch(
                "xagent.web.services.task_orchestrator.load_task_setup_snapshot_sync",
                side_effect=load_snapshot_from_contended_pool,
            ),
            patch(
                "xagent.web.services.task_execution.execute_task_background",
                new=AsyncMock(),
            ) as mock_execute,
            patch(
                "xagent.web.services.task_orchestrator.settle_task_lease_isolated",
            ) as mock_settle,
            patch.object(background_task_manager, "register_task"),
            patch(
                "xagent.web.services.task_orchestrator._get_agent_manager",
                return_value=MagicMock(),
            ),
        ):
            await _schedule_bg(
                task_id=task_id,
                task_owner_user_id=user_id,
                task_source=task_source,
                run_id="run-a",
                payload=TaskTurnPayload("hello"),
                force_fresh=False,
                context=None,
            )

        mock_execute.assert_not_awaited()
        mock_stop_heartbeat.assert_awaited_once()
        mock_settle.assert_not_called()
    finally:
        held_connection.close()

    with SessionLocal() as verify_db:
        retained = verify_db.query(Task).filter(Task.id == task_id).one()
        assert retained.status == TaskStatus.RUNNING
        assert retained.runner_id == lease.runner_id
        assert retained.run_id == lease.run_id

    assert f"task_id={task_id}" in caplog.text
    assert "component=setup/run" in caplog.text
    assert "retaining lease for TTL recovery" in caplog.text


@pytest.mark.asyncio
async def test_schedule_bg_heartbeat_pool_timeout_skips_settlement(
    db_session,
) -> None:
    """A heartbeat checkout timeout must not trigger a cleanup checkout."""
    from xagent.web.services.task_execution import background_task_manager

    user = _create_user(db_session)
    task = _create_task(db_session, user.id, status=TaskStatus.RUNNING)
    task_id = int(task.id)
    lease = TaskLease(
        task_id=task_id,
        runner_id="test-runner",
        run_id="run-a",
        attempt_id="test-attempt",
    )
    heartbeat_timeout = SQLAlchemyTimeoutError("heartbeat pool timeout")

    with (
        patch(
            "xagent.web.services.task_orchestrator.acquire_task_lease_isolated",
            return_value=lease,
        ),
        patch(
            "xagent.web.services.task_orchestrator.run_task_lease_heartbeat",
            new=AsyncMock(),
        ),
        patch(
            "xagent.web.services.task_orchestrator.stop_task_lease_heartbeat",
            new=AsyncMock(
                return_value=TaskLeaseHeartbeatOutcome(pool_timeout=heartbeat_timeout)
            ),
        ),
        patch(
            "xagent.web.services.task_orchestrator.load_task_setup_snapshot_sync",
            return_value=MagicMock(),
        ),
        patch.object(
            task_orchestrator_module,
            "resolve_execution_scope",
            return_value=None,
            create=True,
        ),
        patch(
            "xagent.web.services.task_execution.execute_task_background",
            new=AsyncMock(),
        ),
        patch(
            "xagent.web.services.task_orchestrator.settle_task_lease_isolated",
        ) as settle,
        patch.object(background_task_manager, "register_task"),
        patch(
            "xagent.web.services.task_orchestrator._get_agent_manager",
            return_value=MagicMock(),
        ),
    ):
        await _schedule_bg(
            task_id=task_id,
            task_owner_user_id=int(user.id),
            task_source=task.source,
            run_id="run-a",
            payload=TaskTurnPayload("hello"),
            force_fresh=False,
            context=None,
        )

    settle.assert_not_called()


@pytest.mark.asyncio
async def test_schedule_bg_settles_owned_terminal_task_observed_by_heartbeat(
    db_session,
    monkeypatch,
) -> None:
    """A terminal commit is settlement-ready, not evidence of lease loss."""
    from xagent.web.services.task_execution import background_task_manager

    user = _create_user(db_session)
    agent = Agent(user_id=user.id, name="terminal heartbeat agent")
    db_session.add(agent)
    db_session.flush()
    trigger = AgentTrigger(
        user_id=user.id,
        agent_id=agent.id,
        type=TriggerType.SCHEDULED.value,
        name="terminal heartbeat trigger",
        config={},
    )
    db_session.add(trigger)
    db_session.flush()
    task = _create_task(db_session, user.id, status=TaskStatus.RUNNING)
    task_id = int(task.id)
    task.source = "trigger"
    task.runner_id = "terminal-runner"
    task.lease_attempt_id = "test-attempt"
    task.run_id = "terminal-run"
    task.lease_expires_at = datetime.now(timezone.utc) + timedelta(minutes=5)
    run = TriggerRun(
        trigger_id=trigger.id,
        task_id=task.id,
        status=TriggerRunStatus.RUNNING.value,
        idempotency_key="terminal-heartbeat-race",
    )
    db_session.add(run)
    db_session.commit()

    lease = TaskLease(
        task_id=task_id,
        runner_id="terminal-runner",
        run_id="terminal-run",
        attempt_id="test-attempt",
    )
    terminal_committed = asyncio.Event()
    allow_execution_return = asyncio.Event()

    async def execute_then_wait(**_kwargs) -> None:
        def commit_terminal_status() -> None:
            SessionLocal = database_module.get_session_local()
            with SessionLocal() as terminal_db:
                terminal_task = terminal_db.query(Task).filter(Task.id == task_id).one()
                terminal_task.status = TaskStatus.COMPLETED
                terminal_task.control_state = "completed"
                terminal_db.commit()

        await asyncio.to_thread(commit_terminal_status)
        terminal_committed.set()
        await allow_execution_return.wait()

    from xagent.web.services import task_lease_service

    # Every refresh is a SQLite write. A sub-commit interval keeps the write
    # lock busy back to back and starves settlement's own writes, so leave
    # the lock idle between refreshes and wait for the observed refresh.
    monkeypatch.setattr(
        task_lease_service, "get_task_lease_heartbeat_seconds", lambda: 0.2
    )
    refresh = task_lease_service.refresh_task_leases_isolated
    settlement_observed = Event()

    def observe_refresh(leases):
        states = refresh(leases)
        if TaskLeaseRefreshState.SETTLEMENT_READY in states.values():
            settlement_observed.set()
        return states

    monkeypatch.setattr(
        task_lease_service, "refresh_task_leases_isolated", observe_refresh
    )

    with (
        patch(
            "xagent.web.services.task_orchestrator.acquire_task_lease_isolated",
            return_value=lease,
        ),
        patch(
            "xagent.web.services.task_orchestrator.load_task_setup_snapshot_sync",
            return_value=MagicMock(),
        ),
        patch.object(
            task_orchestrator_module,
            "resolve_execution_scope",
            return_value=None,
            create=True,
        ),
        patch(
            "xagent.web.services.task_execution.execute_task_background",
            new=execute_then_wait,
        ),
        patch.object(background_task_manager, "register_task"),
        patch(
            "xagent.web.services.task_orchestrator._get_agent_manager",
            return_value=MagicMock(),
        ),
    ):
        bg_task = _schedule_bg(
            task_id=task_id,
            task_owner_user_id=int(user.id),
            task_source=task.source,
            run_id="terminal-run",
            payload=TaskTurnPayload("hello"),
            force_fresh=False,
            context=None,
        )
        await asyncio.wait_for(terminal_committed.wait(), timeout=DB_PROGRESS_TIMEOUT)
        # The heartbeat must see the terminal commit while execution still
        # runs; wait for that refresh instead of a fixed sleep.
        assert await asyncio.to_thread(settlement_observed.wait, DB_PROGRESS_TIMEOUT)
        allow_execution_return.set()
        await asyncio.wait_for(bg_task, timeout=DB_PROGRESS_TIMEOUT)

    db_session.expire_all()
    persisted_task = db_session.query(Task).filter(Task.id == task_id).one()
    persisted_run = db_session.query(TriggerRun).filter(TriggerRun.id == run.id).one()
    assert persisted_task.status == TaskStatus.COMPLETED
    assert persisted_task.runner_id is None
    assert persisted_task.lease_expires_at is None
    assert persisted_run.status == TriggerRunStatus.COMPLETED.value


@pytest.mark.asyncio
async def test_schedule_bg_releases_lease_without_blocking_pool_checkout(
    queue_pool_runtime_db_factory,
    monkeypatch,
) -> None:
    """Final status read and workforce-aware release share one worker session."""
    from xagent.web.services.task_execution import background_task_manager

    # The release path must wait for the slot, never give up on it.
    engine, SessionLocal = queue_pool_runtime_db_factory(
        pool_timeout=CONTENTION_POOL_TIMEOUT
    )
    runner_id = get_runner_id()
    with SessionLocal() as seed_db:
        user = _create_user(seed_db)
        task = _create_task(seed_db, user.id, status=TaskStatus.RUNNING)
        task_id = int(task.id)
        user_id = int(user.id)
        task_source = task.source
        task.runner_id = runner_id
        task.lease_attempt_id = "test-attempt"
        task.run_id = "run-a"
        seed_db.commit()

    monkeypatch.setattr(database_module, "_SessionLocal", SessionLocal)
    held_connection = engine.connect()
    ticker_stop = asyncio.Event()
    ticks = 0

    async def ticker() -> None:
        nonlocal ticks
        while not ticker_stop.is_set():
            await asyncio.sleep(0.01)
            ticks += 1

    with (
        gated_pool_checkout(engine) as gate,
        patch(
            "xagent.web.services.task_orchestrator.acquire_task_lease_isolated",
            return_value=TaskLease(
                task_id=task_id,
                runner_id=runner_id,
                run_id="run-a",
                attempt_id="test-attempt",
            ),
        ),
        patch(
            "xagent.web.services.task_orchestrator.run_task_lease_heartbeat",
            new=AsyncMock(),
        ),
        patch(
            "xagent.web.services.task_orchestrator.load_task_setup_snapshot_sync",
            return_value=MagicMock(),
        ),
        patch.object(
            task_orchestrator_module,
            "resolve_execution_scope",
            return_value=None,
            create=True,
        ),
        patch(
            "xagent.web.services.task_execution.execute_task_background",
            new=AsyncMock(),
        ),
        patch.object(background_task_manager, "register_task"),
        patch(
            "xagent.web.services.task_orchestrator._get_agent_manager",
            return_value=MagicMock(),
        ),
    ):
        bg_task = _schedule_bg(
            task_id=task_id,
            task_owner_user_id=user_id,
            task_source=task_source,
            run_id="run-a",
            payload=TaskTurnPayload("hello"),
            force_fresh=False,
            context=None,
        )
        ticker_task = asyncio.create_task(ticker())
        try:
            await gate.wait_until_contending()
            observed = await wait_for_ticks(lambda: ticks)
            assert observed >= LOOP_LIVENESS_TICKS, (
                "lease release QueuePool checkout blocked the loop"
            )
            assert not bg_task.done(), "lease release skipped the contended checkout"
        finally:
            held_connection.close()
            gate.let_through()
            await asyncio.wait_for(bg_task, timeout=GUARD_TIMEOUT)
            ticker_stop.set()
            await ticker_task

    with SessionLocal() as verify_db:
        released = verify_db.query(Task).filter(Task.id == task_id).one()
        assert released.status == TaskStatus.FAILED
        assert released.run_id == "run-a"
        assert released.runner_id is None


@pytest.mark.asyncio
async def test_schedule_bg_releases_lease_on_execute_task_background_exception(
    db_session,
) -> None:
    """Lease must not leak when execute_task_background raises — _runner.finally
    must still call the lease release + workforce sync helper."""
    from xagent.web.services.task_execution import background_task_manager
    from xagent.web.services.task_lease_service import TaskLease

    user = _create_user(db_session)
    task = _create_task(db_session, user.id, status=TaskStatus.RUNNING)
    fake_lease = TaskLease(
        task_id=int(task.id), runner_id="test-runner", attempt_id="test-attempt"
    )
    payload = TaskTurnPayload("x")
    _store_runtime_secret_for_turn(payload.turn_id)
    assert get_ephemeral_runtime_values(payload.turn_id) is not None

    with (
        patch(
            "xagent.web.services.task_orchestrator.acquire_task_lease_isolated",
            return_value=fake_lease,
        ),
        patch(
            "xagent.web.services.task_orchestrator.run_task_lease_heartbeat",
            new=AsyncMock(),
        ),
        patch(
            "xagent.web.services.task_execution.execute_task_background",
            new=AsyncMock(side_effect=RuntimeError("boom")),
        ),
        patch(
            "xagent.web.services.task_orchestrator.settle_task_lease_isolated",
        ) as mock_settle,
        patch.object(background_task_manager, "register_task"),
        patch(
            "xagent.web.services.task_orchestrator._get_agent_manager",
            return_value=MagicMock(),
        ),
    ):
        bg_task = _schedule_bg(
            task_id=int(task.id),
            task_owner_user_id=int(user.id),
            task_source=task.source,
            payload=payload,
            force_fresh=False,
            context=None,
        )
        # Wait for the inner _runner to finish (which raises internally
        # but the wrapping create_task absorbs it). The release should
        # still have been called in _runner.finally.
        try:
            await bg_task
        except RuntimeError:
            pass

    mock_settle.assert_called_once()
    assert get_ephemeral_runtime_values(payload.turn_id) is None
    assert pop_ephemeral_runtime_values(payload.turn_id) is None


@pytest.mark.parametrize(
    ("settled", "expected_broadcasts"),
    [(True, 1), (False, 0)],
)
@pytest.mark.asyncio
async def test_schedule_bg_broadcasts_failure_only_after_exact_settlement(
    db_session,
    settled: bool,
    expected_broadcasts: int,
) -> None:
    """A replacement owner must never inherit a stale runner's error event."""
    from xagent.web.services.task_execution import background_task_manager

    user = _create_user(db_session)
    task = _create_task(db_session, user.id, status=TaskStatus.RUNNING)
    task_id = int(task.id)
    lease = TaskLease(
        task_id=task_id, runner_id="runner-a", run_id="run-a", attempt_id="test-attempt"
    )
    events: list[str] = []

    def settle(*_args, **_kwargs) -> bool:
        events.append("settle")
        return settled

    async def broadcast(*_args, **_kwargs) -> None:
        events.append("broadcast")

    with (
        patch(
            "xagent.web.services.task_orchestrator.acquire_task_lease_isolated",
            return_value=lease,
        ),
        patch(
            "xagent.web.services.task_orchestrator.run_task_lease_heartbeat",
            new=AsyncMock(),
        ),
        patch(
            "xagent.web.services.task_orchestrator.load_task_setup_snapshot_sync",
            return_value=MagicMock(),
        ),
        patch.object(
            task_orchestrator_module,
            "resolve_execution_scope",
            return_value=None,
            create=True,
        ),
        patch(
            "xagent.web.services.task_execution.execute_task_background",
            new=AsyncMock(side_effect=RuntimeError("owned run failed")),
        ),
        patch(
            "xagent.web.services.task_orchestrator.settle_task_lease_isolated",
            side_effect=settle,
        ),
        patch(
            "xagent.web.api.websocket.manager",
            MagicMock(broadcast_to_task=AsyncMock(side_effect=broadcast)),
        ),
        patch.object(background_task_manager, "register_task"),
        patch(
            "xagent.web.services.task_orchestrator._get_agent_manager",
            return_value=MagicMock(),
        ),
    ):
        await _schedule_bg(
            task_id=task_id,
            task_owner_user_id=int(user.id),
            task_source=task.source,
            payload=TaskTurnPayload("hello"),
            force_fresh=False,
            context=None,
        )

    assert events == ["settle"] + (["broadcast"] * expected_broadcasts)


@pytest.mark.asyncio
async def test_marked_append_rejects_before_persisting_turn(db_session) -> None:
    user = _create_user(db_session)
    task = _create_task(db_session, user.id, status=TaskStatus.COMPLETED)
    task.agent_config = {MCP_RUNTIME_AUTHORIZATION_POLICY_REQUIRED_KEY: True}
    original_run_id = task.run_id
    original_state_version = task.state_version
    db_session.commit()

    with pytest.raises(TaskTurnError, match="actor_task_reuse_unsupported"):
        await TaskTurnOrchestrator.begin_turn(
            task_id=int(task.id),
            task_owner_user_id=int(user.id),
            payload=TaskTurnPayload("generic append"),
            kind=TurnKind.APPEND,
        )

    db_session.expire_all()
    persisted = db_session.get(Task, int(task.id))
    assert persisted is not None
    assert persisted.status == TaskStatus.COMPLETED
    assert persisted.run_id == original_run_id
    assert persisted.state_version == original_state_version
    assert (
        db_session.query(TaskChatMessage)
        .filter(TaskChatMessage.task_id == int(task.id))
        .count()
        == 0
    )


@pytest.mark.asyncio
async def test_trusted_marked_create_schedule_forwards_actor_policy() -> None:
    policy = MCPBuiltinOAuthActorPolicy(resource_owner_key="actor:alice")
    lease = TaskLease(
        task_id=42,
        runner_id="trusted-direct",
        run_id="run-42",
        attempt_id="test-attempt",
    )
    claimed = _ClaimedTurn(
        task_lease=lease,
        status=TaskStatus.RUNNING,
        updated_at=None,
        before_message_id=None,
        task_source="external",
        run_id="run-42",
        agent_config={MCP_RUNTIME_AUTHORIZATION_POLICY_REQUIRED_KEY: True},
    )

    async def completed() -> None: ...

    background = asyncio.create_task(completed())
    with (
        patch(
            "xagent.web.services.task_orchestrator.invalidate_task_cache_best_effort"
        ),
        patch(
            "xagent.web.services.task_orchestrator._schedule_bg",
            return_value=background,
        ) as schedule,
    ):
        started = await TaskTurnOrchestrator.schedule_claimed_create_turn(
            task_id=42,
            task_owner_user_id=7,
            actor_user_id=7,
            payload=TaskTurnPayload("fresh actor message"),
            claimed=claimed,
            mcp_runtime_authorization_policy=policy,
        )

    assert started.background_task is background
    assert schedule.call_args.kwargs["mcp_runtime_authorization_policy"] is policy
    await background


@pytest.mark.asyncio
async def test_schedule_bg_forwards_execution_message_to_execute_task_background(
    db_session,
) -> None:
    """Dual-channel payload propagation through the scheduler:
    ``_schedule_bg`` must pass ``payload.execution_message`` to
    ``execute_task_background``'s ``llm_user_message=`` parameter so
    the LLM-facing variant of the turn input survives the orchestrator
    boundary.

    Together with the ``begin_turn → _schedule_bg`` test above this
    locks in the full payload chain
    (begin_turn → _schedule_bg → execute_task_background) at the
    type-signature level, so a future refactor can't silently collapse
    transcript and execution into a single string.
    """
    from xagent.web.services.task_execution import background_task_manager
    from xagent.web.services.task_lease_service import TaskLease

    user = _create_user(db_session)
    task = _create_task(db_session, user.id, status=TaskStatus.RUNNING)
    fake_lease = TaskLease(
        task_id=int(task.id), runner_id="test-runner", attempt_id="test-attempt"
    )
    payload = TaskTurnPayload(
        transcript_message="summarize this",
        execution_message="summarize this\n\n[uploaded file: secret.txt]",
    )
    _store_runtime_secret_for_turn(payload.turn_id)
    assert get_ephemeral_runtime_values(payload.turn_id) is not None
    actor_policy = MCPBuiltinOAuthActorPolicy(resource_owner_key="actor:alice")
    agent_manager = MagicMock()

    with (
        patch(
            "xagent.web.services.task_orchestrator.acquire_task_lease_isolated",
            return_value=fake_lease,
        ),
        patch(
            "xagent.web.services.task_orchestrator.run_task_lease_heartbeat",
            new=AsyncMock(),
        ),
        patch(
            "xagent.web.services.task_execution.execute_task_background",
            new=AsyncMock(),
        ) as mock_exec,
        patch(
            "xagent.web.services.task_orchestrator.settle_task_lease_isolated",
        ),
        patch.object(background_task_manager, "register_task"),
        patch(
            "xagent.web.services.task_orchestrator._get_agent_manager",
            return_value=agent_manager,
        ),
    ):
        bg_task = _schedule_bg(
            task_id=int(task.id),
            task_owner_user_id=int(user.id),
            task_source=task.source,
            payload=payload,
            force_fresh=False,
            context={"turn_id": "caller-turn", "existing": "value"},
            mcp_runtime_authorization_policy=actor_policy,
        )
        await bg_task

    mock_exec.assert_awaited_once()
    kwargs = mock_exec.await_args.kwargs
    # Dual-channel payload contract: transcript and LLM-facing channels are both
    # forwarded explicitly so execute_task_background can pick the
    # right one for the agent input.
    assert kwargs["user_message"] == "summarize this", (
        "transcript_message must reach execute_task_background.user_message"
    )
    assert (
        kwargs["llm_user_message"] == "summarize this\n\n[uploaded file: secret.txt]"
    ), "execution_message must reach execute_task_background.llm_user_message"
    assert kwargs["context"]["turn_id"] == payload.turn_id
    assert kwargs["context"]["existing"] == "value"
    assert kwargs["mcp_runtime_authorization_policy"] is actor_policy
    agent_manager.remove_agent.assert_called_once_with(
        int(task.id),
        int(user.id),
        expected_run_id=fake_lease.run_id,
    )
    assert get_ephemeral_runtime_values(payload.turn_id) is None
    assert pop_ephemeral_runtime_values(payload.turn_id) is None


@pytest.mark.asyncio
async def test_schedule_bg_acquires_expired_lease_on_first_try(db_session) -> None:
    """Expired lease columns are granted by acquire_task_lease's atomic WHERE."""
    from xagent.web.services.task_execution import background_task_manager
    from xagent.web.services.task_lease_service import utc_now

    user = _create_user(db_session)
    task = _create_task(db_session, user.id, status=TaskStatus.RUNNING)
    task.runner_id = "dead-runner"
    task.lease_attempt_id = "test-attempt"
    task.lease_expires_at = utc_now() - timedelta(seconds=5)
    db_session.commit()

    fake_snapshot = MagicMock()

    with (
        patch(
            "xagent.web.services.task_orchestrator.run_task_lease_heartbeat",
            new=AsyncMock(),
        ),
        patch(
            "xagent.web.services.task_orchestrator.load_task_setup_snapshot_sync",
            return_value=fake_snapshot,
        ),
        patch(
            "xagent.web.services.task_execution.execute_task_background",
            new=AsyncMock(),
        ) as mock_exec,
        patch(
            "xagent.web.services.task_orchestrator.settle_task_lease_isolated",
        ),
        patch.object(background_task_manager, "register_task"),
        patch(
            "xagent.web.services.task_orchestrator._get_agent_manager",
            return_value=MagicMock(),
        ),
    ):
        bg_task = _schedule_bg(
            task_id=int(task.id),
            task_owner_user_id=int(user.id),
            task_source=task.source,
            payload=TaskTurnPayload("hello"),
            force_fresh=False,
            context=None,
        )
        await bg_task

    mock_exec.assert_awaited_once()


# ---------------------------------------------------------------------------
# _runner setup-error → FAILED safety net: prevents the
# acquire_lease-sets-RUNNING-then-no-one-clears-it zombie state when
# snapshot load or execute_task_background raises.
# ---------------------------------------------------------------------------


@pytest.mark.asyncio
async def test_schedule_bg_marks_task_failed_when_snapshot_load_raises(
    db_session,
) -> None:
    """Snapshot-load exception must not leave the row visible-as-running.

    ``acquire_task_lease_isolated`` writes ``status=RUNNING`` as part
    of taking the lease. Without the outer ``except`` in ``_runner``,
    an exception out of ``load_task_setup_snapshot_sync`` propagates
    through ``_runner``'s inner ``try`` block; ``finish_turn`` and
    ``execute_task_background`` never run, and the outer release
    block reads the still-RUNNING status and writes it back --
    leaving the task displayed as running but with no worker
    executing it.

    The outer ``except`` records the error and the one fenced settlement
    transaction pushes the exact run to ``FAILED`` while releasing it.
    """
    from xagent.web.api.websocket import manager as ws_manager
    from xagent.web.services.task_execution import background_task_manager
    from xagent.web.services.task_lease_service import TaskLease

    user = _create_user(db_session)
    task = _create_task(db_session, user.id, status=TaskStatus.RUNNING)
    task.runner_id = "test-runner"
    task.lease_attempt_id = "test-attempt"
    task.run_id = "run-a"
    db_session.commit()
    fake_lease = TaskLease(
        task_id=int(task.id),
        runner_id="test-runner",
        run_id="run-a",
        attempt_id="test-attempt",
    )
    payload = TaskTurnPayload("x")
    _store_runtime_secret_for_turn(payload.turn_id)
    assert get_ephemeral_runtime_values(payload.turn_id) is not None

    with (
        patch(
            "xagent.web.services.task_orchestrator.acquire_task_lease_isolated",
            return_value=fake_lease,
        ),
        patch(
            "xagent.web.services.task_orchestrator.run_task_lease_heartbeat",
            new=AsyncMock(),
        ),
        patch(
            "xagent.web.services.task_orchestrator.load_task_setup_snapshot_sync",
            side_effect=RuntimeError("simulated snapshot load failure"),
        ),
        patch(
            "xagent.web.services.task_execution.execute_task_background",
            new=AsyncMock(),
        ) as mock_exec,
        patch.object(background_task_manager, "register_task"),
        patch.object(ws_manager, "broadcast_to_task", new=AsyncMock()) as broadcast,
        patch(
            "xagent.web.services.task_orchestrator._get_agent_manager",
            return_value=MagicMock(),
        ),
    ):
        bg_task = _schedule_bg(
            task_id=int(task.id),
            task_owner_user_id=int(user.id),
            task_source=task.source,
            payload=payload,
            force_fresh=False,
            context=None,
        )
        try:
            await bg_task
        except RuntimeError:
            pass

    # execute_task_background must not run when snapshot load raised.
    mock_exec.assert_not_called()
    # The row should now be FAILED, not the zombie RUNNING.
    db_session.refresh(task)
    assert task.status == TaskStatus.FAILED, (
        f"Expected task.status == FAILED after snapshot raise, got {task.status}. "
        "If this fails, the fenced settlement did not close the zombie-RUNNING "
        "window."
    )
    assert task.runner_id is None
    assert task.error_message is not None
    assert "simulated snapshot load failure" in str(task.error_message)
    task_errors = [
        call.args[0]
        for call in broadcast.call_args_list
        if call.args[0].get("type") == "task_error"
    ]
    assert len(task_errors) == 1
    assert task_errors[0]["message"] == "Task execution failed."
    assert task_errors[0]["error"] == "Task execution failed."
    assert "simulated snapshot load failure" not in repr(task_errors[0])
    assert get_ephemeral_runtime_values(payload.turn_id) is None
    assert pop_ephemeral_runtime_values(payload.turn_id) is None


@pytest.mark.asyncio
async def test_schedule_bg_cleanup_handles_missing_payload_turn_id(db_session) -> None:
    from xagent.web.services.task_execution import background_task_manager
    from xagent.web.services.task_lease_service import TaskLease

    user = _create_user(db_session)
    task = _create_task(db_session, user.id, status=TaskStatus.RUNNING)
    fake_lease = TaskLease(
        task_id=int(task.id), runner_id="test-runner", attempt_id="test-attempt"
    )

    with (
        patch(
            "xagent.web.services.task_orchestrator.acquire_task_lease_isolated",
            return_value=fake_lease,
        ),
        patch(
            "xagent.web.services.task_orchestrator.run_task_lease_heartbeat",
            new=AsyncMock(),
        ),
        patch(
            "xagent.web.services.task_orchestrator.load_task_setup_snapshot_sync",
            return_value=MagicMock(),
        ),
        patch(
            "xagent.web.services.task_execution.execute_task_background",
            new=AsyncMock(),
        ) as mock_exec,
        patch(
            "xagent.web.services.task_orchestrator.settle_task_lease_isolated",
        ),
        patch.object(background_task_manager, "register_task"),
        patch(
            "xagent.web.services.task_orchestrator._get_agent_manager",
            return_value=MagicMock(),
        ),
    ):
        bg_task = _schedule_bg(
            task_id=int(task.id),
            task_owner_user_id=int(user.id),
            task_source=task.source,
            payload=None,  # type: ignore[arg-type]
            force_fresh=False,
            context=None,
        )
        await bg_task

    mock_exec.assert_not_called()


@pytest.mark.asyncio
@pytest.mark.parametrize("source", ["web", "trigger"])
async def test_schedule_bg_preserves_public_safe_required_mcp_failure(
    db_session,
    monkeypatch,
    source,
) -> None:
    """A typed, curated setup failure remains actionable to the client."""
    from xagent.core.utils import setup_metrics
    from xagent.web.api.websocket import manager as ws_manager
    from xagent.web.services.task_execution import background_task_manager

    counters = setup_metrics.SetupMetrics()
    monkeypatch.setattr(setup_metrics, "trigger_execution", counters)
    user = _create_user(db_session)
    task = _create_task(db_session, user.id, status=TaskStatus.RUNNING)
    task.runner_id = "test-runner"
    task.lease_attempt_id = "test-attempt"
    task.run_id = "run-a"
    db_session.commit()
    lease = TaskLease(
        task_id=int(task.id),
        runner_id="test-runner",
        run_id="run-a",
        attempt_id="test-attempt",
    )
    public_message = "Required MCP servers are unavailable: mcp_server_unavailable."

    with (
        patch(
            "xagent.web.services.task_orchestrator.acquire_task_lease_isolated",
            return_value=lease,
        ),
        patch(
            "xagent.web.services.task_orchestrator.run_task_lease_heartbeat",
            new=AsyncMock(),
        ),
        patch(
            "xagent.web.services.task_orchestrator.load_task_setup_snapshot_sync",
            side_effect=RequiredMCPUnavailableError([]),
        ),
        patch.object(background_task_manager, "register_task"),
        patch.object(ws_manager, "broadcast_to_task", new=AsyncMock()) as broadcast,
        patch(
            "xagent.web.services.task_orchestrator._get_agent_manager",
            return_value=MagicMock(),
        ),
    ):
        background = _schedule_bg(
            task_id=int(task.id),
            task_owner_user_id=int(user.id),
            task_source=source,
            payload=TaskTurnPayload("hello"),
            force_fresh=False,
            context=None,
        )

        assert counters.active == (source == "trigger")
        await background
        assert counters.active == 0
        assert counters.completed == (source == "trigger")
        assert counters.failed == (source == "trigger")
        assert counters.cancelled == 0

    db_session.expire_all()
    persisted = db_session.get(Task, int(task.id))
    assert persisted is not None
    assert persisted.error_message == public_message
    assistant = (
        db_session.query(TaskChatMessage)
        .filter(
            TaskChatMessage.task_id == task.id,
            TaskChatMessage.role == "assistant",
        )
        .one()
    )
    assert assistant.content == public_message
    assert assistant.message_type == CLIENT_SAFE_FAILURE_MESSAGE_TYPE
    broadcast.assert_awaited_once()
    event = broadcast.await_args.args[0]
    assert event["message"] == public_message
    assert event["error"] == public_message


@pytest.mark.asyncio
async def test_schedule_bg_marks_task_failed_when_execute_raises(
    db_session,
) -> None:
    """An execution exception is persisted by the one fenced settlement."""
    from xagent.web.api.websocket import manager as ws_manager
    from xagent.web.services.task_execution import background_task_manager
    from xagent.web.services.task_lease_service import TaskLease

    user = _create_user(db_session)
    task = _create_task(db_session, user.id, status=TaskStatus.RUNNING)
    task.runner_id = "test-runner"
    task.lease_attempt_id = "test-attempt"
    task.run_id = "run-a"
    db_session.commit()
    fake_lease = TaskLease(
        task_id=int(task.id),
        runner_id="test-runner",
        run_id="run-a",
        attempt_id="test-attempt",
    )

    # Snapshot loader returns a minimal sentinel snapshot so the test
    # proceeds past the snapshot-None branch.
    fake_snapshot = MagicMock()

    with (
        patch(
            "xagent.web.services.task_orchestrator.acquire_task_lease_isolated",
            return_value=fake_lease,
        ),
        patch(
            "xagent.web.services.task_orchestrator.run_task_lease_heartbeat",
            new=AsyncMock(),
        ),
        patch(
            "xagent.web.services.task_orchestrator.load_task_setup_snapshot_sync",
            return_value=fake_snapshot,
        ),
        patch(
            "xagent.web.services.task_execution.execute_task_background",
            new=AsyncMock(side_effect=RuntimeError("simulated agent boom")),
        ),
        patch.object(background_task_manager, "register_task"),
        patch.object(ws_manager, "broadcast_to_task", new=AsyncMock()) as broadcast,
        patch(
            "xagent.web.services.task_orchestrator._get_agent_manager",
            return_value=MagicMock(),
        ),
    ):
        bg_task = _schedule_bg(
            task_id=int(task.id),
            task_owner_user_id=int(user.id),
            task_source=task.source,
            payload=TaskTurnPayload("x"),
            force_fresh=False,
            context=None,
        )
        try:
            await bg_task
        except RuntimeError:
            pass

    db_session.refresh(task)
    assert task.status == TaskStatus.FAILED
    assert task.error_message is not None
    assert "simulated agent boom" in str(task.error_message)
    broadcast.assert_awaited_once()
    event = broadcast.await_args.args[0]
    assert event["message"] == "Task execution failed."
    assert event["error"] == "Task execution failed."
    assert "simulated agent boom" not in repr(event)


@pytest.mark.asyncio
async def test_schedule_bg_does_not_overwrite_terminal_status_from_execute(
    db_session,
) -> None:
    """Fenced settlement never overwrites a committed control status.

    Simulate the inner handler setting PAUSED before raising. After
    ``_runner`` returns, the row must remain PAUSED, not be flipped
    to FAILED by the outer safety net.
    """
    from xagent.web.services.task_execution import background_task_manager
    from xagent.web.services.task_lease_service import TaskLease

    user = _create_user(db_session)
    task = _create_task(db_session, user.id, status=TaskStatus.RUNNING)
    task.runner_id = "test-runner"
    task.lease_attempt_id = "test-attempt"
    task.run_id = "run-a"
    db_session.commit()
    fake_lease = TaskLease(
        task_id=int(task.id),
        runner_id="test-runner",
        run_id="run-a",
        attempt_id="test-attempt",
    )
    fake_snapshot = MagicMock()

    async def fake_execute(*args, **kwargs):
        # Inner handler decides the turn is paused, commits, then a
        # later step raises. Outer except must not undo the PAUSED.
        from xagent.web.models.task import Task as TaskModel

        with sessionmaker(bind=get_engine())() as inner:
            row = inner.query(TaskModel).filter(TaskModel.id == task.id).first()
            row.status = TaskStatus.PAUSED
            inner.commit()
        raise RuntimeError("simulated late-stage error after PAUSED")

    with (
        patch(
            "xagent.web.services.task_orchestrator.acquire_task_lease_isolated",
            return_value=fake_lease,
        ),
        patch(
            "xagent.web.services.task_orchestrator.run_task_lease_heartbeat",
            new=AsyncMock(),
        ),
        patch(
            "xagent.web.services.task_orchestrator.load_task_setup_snapshot_sync",
            return_value=fake_snapshot,
        ),
        patch(
            "xagent.web.services.task_execution.execute_task_background",
            new=fake_execute,
        ),
        patch.object(background_task_manager, "register_task"),
        patch(
            "xagent.web.services.task_orchestrator._get_agent_manager",
            return_value=MagicMock(),
        ),
    ):
        bg_task = _schedule_bg(
            task_id=int(task.id),
            task_owner_user_id=int(user.id),
            task_source=task.source,
            payload=TaskTurnPayload("x"),
            force_fresh=False,
            context=None,
        )
        try:
            await bg_task
        except RuntimeError:
            pass

    db_session.refresh(task)
    assert task.status == TaskStatus.PAUSED, (
        f"Expected PAUSED (set by execute), got {task.status}. If this fails, "
        "the concrete-lease settlement overwrote a control status."
    )
    assert task.runner_id is None


# ---------------------------------------------------------------------------
# Finalize-time delivery reconciliation (#332)
# ---------------------------------------------------------------------------


def _claim_pending_delivery(db, *, task_id: int, user_id: int, turn_id: str) -> None:
    """Create a committed ``pending`` delivery row, mirroring a turn claim."""

    claim = claim_user_message_delivery_no_commit(
        db,
        task_id,
        user_id,
        "hello there",
        turn_id=turn_id,
    )
    db.commit()
    assert claim.claimed is True
    assert claim.pending is True


def _delivery_status(db, turn_id: str) -> str | None:
    db.expire_all()
    row = (
        db.query(TaskChatMessage)
        .filter(
            TaskChatMessage.turn_id == turn_id,
            TaskChatMessage.role == "user",
        )
        .one_or_none()
    )
    return None if row is None else str(row.delivery_status)


def test_reconcile_finalized_delivery_completes_orphaned_pending_on_success(
    db_session,
) -> None:
    """A clean settlement advances an orphaned ``pending`` row to ``completed``."""
    user = _create_user(db_session)
    task = _create_task(db_session, int(user.id), status=TaskStatus.RUNNING)
    _claim_pending_delivery(
        db_session, task_id=int(task.id), user_id=int(user.id), turn_id="turn-ok"
    )

    _reconcile_finalized_turn_delivery(
        task_id=int(task.id),
        turn_id="turn-ok",
        settlement_error=None,
        execution_started=True,
    )

    assert _delivery_status(db_session, "turn-ok") == DELIVERY_COMPLETED


def test_reconcile_finalized_delivery_fails_orphaned_pending_when_never_ran(
    db_session,
) -> None:
    """A turn that provably never executed advances ``pending`` to ``failed``.

    ``failed`` invites a fresh-id retry downstream, so it is only written with
    positive evidence the message was never consumed
    (``execution_started=False``). That is the row a same-``client_message_id``
    retry needs: a usable terminal ack instead of an in-flight ``PENDING``
    that is rejected forever.
    """
    user = _create_user(db_session)
    task = _create_task(db_session, int(user.id), status=TaskStatus.RUNNING)
    _claim_pending_delivery(
        db_session, task_id=int(task.id), user_id=int(user.id), turn_id="turn-boom"
    )

    _reconcile_finalized_turn_delivery(
        task_id=int(task.id),
        turn_id="turn-boom",
        settlement_error="setup/run error: RuntimeError: boom",
        execution_started=False,
    )

    assert _delivery_status(db_session, "turn-boom") == DELIVERY_FAILED
    # The probe's ``pending`` predicate — what rejected the retry forever — is
    # now False, so the client can converge.
    inspected = inspect_user_message_delivery(
        db_session,
        int(task.id),
        "hello there",
        attachments=None,
        turn_id="turn-boom",
    )
    assert inspected is not None
    assert inspected.pending is False
    assert inspected.failed is True


def test_reconcile_finalized_delivery_marks_dispatched_when_run_failed_after_start(
    db_session,
) -> None:
    """A failure after execution began closes ``pending`` as ``dispatched``.

    The run may have consumed the message (and produced side effects) before
    failing, so writing ``failed`` would invite a fresh-id retry and a double
    execution. ``dispatched`` converges the client without a resend; the
    turn's failure is surfaced through task status.
    """
    user = _create_user(db_session)
    task = _create_task(db_session, int(user.id), status=TaskStatus.RUNNING)
    _claim_pending_delivery(
        db_session, task_id=int(task.id), user_id=int(user.id), turn_id="turn-late"
    )

    _reconcile_finalized_turn_delivery(
        task_id=int(task.id),
        turn_id="turn-late",
        settlement_error="setup/run error: RuntimeError: late boom",
        execution_started=True,
    )

    assert _delivery_status(db_session, "turn-late") == DELIVERY_DISPATCHED


def test_reconcile_finalized_delivery_noop_without_turn_id(db_session) -> None:
    """A turn with no durable delivery row (no turn id) is a safe no-op."""
    user = _create_user(db_session)
    task = _create_task(db_session, int(user.id), status=TaskStatus.RUNNING)

    # Must not raise even though there is nothing to reconcile.
    _reconcile_finalized_turn_delivery(
        task_id=int(task.id),
        turn_id=None,
        settlement_error=None,
        execution_started=True,
    )


@pytest.mark.parametrize("execution_started", [True, False])
def test_reconcile_finalized_delivery_does_not_regress_dispatched_on_error(
    db_session,
    execution_started,
) -> None:
    """``dispatched`` is terminal for dedup; a later failure must not regress it.

    A normally dispatched turn already gave the client a usable ack. With
    ``execution_started=True`` the ``dispatched`` target is idempotent; with
    ``False`` the monotonic state machine rejects ``dispatched -> failed``.
    Either way the row stays ``dispatched`` and the turn's failure is surfaced
    through task status, not by reopening delivery.
    """
    user = _create_user(db_session)
    task = _create_task(db_session, int(user.id), status=TaskStatus.RUNNING)
    _claim_pending_delivery(
        db_session, task_id=int(task.id), user_id=int(user.id), turn_id="turn-disp"
    )
    transition = mark_user_message_delivery(
        db_session,
        task_id=int(task.id),
        turn_id="turn-disp",
        status=DELIVERY_DISPATCHED,
    )
    assert transition.outcome == "updated"
    db_session.commit()
    db_session.expire_all()

    _reconcile_finalized_turn_delivery(
        task_id=int(task.id),
        turn_id="turn-disp",
        settlement_error="boom",
        execution_started=execution_started,
    )

    assert _delivery_status(db_session, "turn-disp") == DELIVERY_DISPATCHED


def test_reconcile_finalized_delivery_completes_dispatched_on_success(
    db_session,
) -> None:
    """A clean settlement closes a ``dispatched`` row to ``completed``.

    The orchestrator path never marked ``completed`` before; finalize now
    closes the lifecycle to match the resume path.
    """
    user = _create_user(db_session)
    task = _create_task(db_session, int(user.id), status=TaskStatus.RUNNING)
    _claim_pending_delivery(
        db_session, task_id=int(task.id), user_id=int(user.id), turn_id="turn-done"
    )
    mark_user_message_delivery(
        db_session,
        task_id=int(task.id),
        turn_id="turn-done",
        status=DELIVERY_DISPATCHED,
    )
    db_session.commit()

    _reconcile_finalized_turn_delivery(
        task_id=int(task.id),
        turn_id="turn-done",
        settlement_error=None,
        execution_started=True,
    )

    assert _delivery_status(db_session, "turn-done") == DELIVERY_COMPLETED


def test_reconcile_finalized_delivery_swallows_pool_timeout_leaving_pending(
    db_session,
    caplog,
) -> None:
    """A pool timeout leaves the row ``pending`` for durable recovery, no raise."""
    user = _create_user(db_session)
    task = _create_task(db_session, int(user.id), status=TaskStatus.RUNNING)
    _claim_pending_delivery(
        db_session, task_id=int(task.id), user_id=int(user.id), turn_id="turn-pool"
    )

    caplog.set_level(logging.ERROR, logger="xagent.web.services.task_orchestrator")
    with patch(
        "xagent.web.services.task_orchestrator.mark_user_message_delivery_sync",
        side_effect=SQLAlchemyTimeoutError("finalize delivery pool exhausted"),
    ):
        _reconcile_finalized_turn_delivery(
            task_id=int(task.id),
            turn_id="turn-pool",
            settlement_error=None,
            execution_started=True,
        )

    assert _delivery_status(db_session, "turn-pool") == DELIVERY_PENDING
    assert "component=turn-finalize-delivery" in caplog.text


def _finalize_turn_fixture(db_session, *, turn_id: str):
    """One RUNNING task with a committed lease and an orphaned ``pending`` row.

    Mirrors the #332 failure state: the turn claim committed (task RUNNING,
    lease owned, delivery ``pending``) but the ``dispatched`` projection never
    landed.
    """
    user = _create_user(db_session)
    task = _create_task(db_session, int(user.id), status=TaskStatus.RUNNING)
    payload = TaskTurnPayload("hello there", turn_id=turn_id)
    _claim_pending_delivery(
        db_session, task_id=int(task.id), user_id=int(user.id), turn_id=turn_id
    )
    task.runner_id = "test-runner"
    task.lease_attempt_id = "test-attempt"
    task.run_id = "run-a"
    db_session.commit()
    lease = TaskLease(
        task_id=int(task.id),
        runner_id="test-runner",
        run_id="run-a",
        attempt_id="test-attempt",
    )
    return user, task, payload, lease


@contextmanager
def _finalize_runner_patches(
    fake_lease: TaskLease,
    *,
    execute=None,
    settle=None,
    mark_delivery=None,
    validate=None,
    snapshot=None,
    heartbeat=None,
):
    """Patch scaffolding for driving ``_schedule_bg``'s ``_runner`` end-to-end.

    Only the agent runtime is faked by default; lease settlement and delivery
    marking run for real against SQLite unless a stub is supplied.

    Pass a heartbeat stub here rather than patching ``run_task_lease_heartbeat``
    again inside the block: a second patcher of the same attribute that is not
    undone in LIFO order (``monkeypatch`` undoes at teardown, after this block
    exits) restores this block's ``AsyncMock`` and leaks it into every later
    test on the worker.
    """
    from xagent.web.services.task_execution import background_task_manager

    patches = [
        patch(
            "xagent.web.services.task_orchestrator.acquire_task_lease_isolated",
            return_value=fake_lease,
        ),
        patch(
            "xagent.web.services.task_orchestrator.run_task_lease_heartbeat",
            new=heartbeat if heartbeat is not None else AsyncMock(),
        ),
        patch(
            "xagent.web.services.task_orchestrator.load_task_setup_snapshot_sync",
            new=snapshot
            if snapshot is not None
            else MagicMock(return_value=MagicMock()),
        ),
        patch(
            "xagent.web.services.task_execution.execute_task_background",
            new=execute if execute is not None else AsyncMock(),
        ),
        patch.object(background_task_manager, "register_task"),
        patch(
            "xagent.web.services.task_orchestrator._get_agent_manager",
            return_value=MagicMock(),
        ),
    ]
    if settle is not None:
        patches.append(
            patch(
                "xagent.web.services.task_orchestrator.settle_task_lease_isolated",
                settle,
            )
        )
    if mark_delivery is not None:
        patches.append(
            patch(
                "xagent.web.services.task_orchestrator.mark_user_message_delivery_sync",
                mark_delivery,
            )
        )
    if validate is not None:
        patches.append(
            patch(
                "xagent.web.services.task_orchestrator."
                "validate_preacquired_task_lease_isolated",
                validate,
            )
        )
    with ExitStack() as stack:
        for entry in patches:
            stack.enter_context(entry)
        yield


def _spawn_finalize_runner(task, user, payload, **schedule_kwargs):
    return _schedule_bg(
        task_id=int(task.id),
        task_owner_user_id=int(user.id),
        task_source=task.source,
        payload=payload,
        force_fresh=False,
        context=None,
        **schedule_kwargs,
    )


@pytest.mark.asyncio
async def test_shared_scheduler_skips_local_runtime_input_store(
    db_session, monkeypatch
) -> None:
    user, task, payload, lease = _finalize_turn_fixture(
        db_session, turn_id="shared-runtime-inputs"
    )
    monkeypatch.setattr(
        task_orchestrator_module, "get_shared_task_execution_enabled", lambda: True
    )
    with (
        _finalize_runner_patches(lease, settle=MagicMock(return_value=True)),
        patch(
            "xagent.web.services.connector_runtime.bind_ephemeral_runtime_values_to_run"
        ) as bind,
        patch(
            "xagent.web.services.connector_runtime.clean_ephemeral_runtime_values_for_run"
        ) as clean,
        patch(
            "xagent.web.services.task_runtime_secrets.clean_finished_runtime_values"
        ) as shared_clean,
    ):
        await _spawn_finalize_runner(task, user, payload, run_id=lease.run_id)

    bind.assert_not_called()
    clean.assert_not_called()
    shared_clean.assert_called_once_with(task_id=int(task.id), run_id=lease.run_id)


@pytest.mark.asyncio
async def test_runner_finalize_reconciles_orphaned_pending_delivery(
    db_session,
) -> None:
    """End-to-end: a turn whose ``dispatched`` projection never landed still
    finalizes its delivery row to ``completed`` when the run settles cleanly."""
    user, task, payload, lease = _finalize_turn_fixture(
        db_session, turn_id="turn-e2e-ok"
    )

    with _finalize_runner_patches(lease):
        await _spawn_finalize_runner(task, user, payload)

    assert _delivery_status(db_session, "turn-e2e-ok") == DELIVERY_COMPLETED


@pytest.mark.asyncio
async def test_runner_finalize_reconciles_alongside_a_genuinely_completed_turn(
    db_session,
) -> None:
    """A real success settles through ``finish_turn``'s COMPLETED branch and
    closes the orphaned delivery row in the same finalize.

    The sibling success test reaches ``completed`` delivery via
    ``settlement_error is None`` while the task itself lands FAILED (its mocked
    runtime never signals success), so it cannot catch a regression in
    ``finish_turn``'s COMPLETED branch. Here the fake runtime commits
    COMPLETED plus an assistant message — what that branch reads to write
    ``output`` — so task terminalization and delivery reconciliation are
    asserted together.
    """
    user, task, payload, lease = _finalize_turn_fixture(
        db_session, turn_id="turn-e2e-real-success"
    )

    async def succeeding_execute(*args, **kwargs):
        with sessionmaker(bind=get_engine())() as inner:
            row = inner.query(Task).filter(Task.id == task.id).one()
            row.status = TaskStatus.COMPLETED
            inner.add(
                TaskChatMessage(
                    task_id=int(task.id),
                    user_id=int(user.id),
                    role="assistant",
                    content="here is your answer",
                    message_type="chat_response",
                )
            )
            inner.commit()

    with _finalize_runner_patches(lease, execute=succeeding_execute):
        await _spawn_finalize_runner(task, user, payload)

    assert _delivery_status(db_session, "turn-e2e-real-success") == DELIVERY_COMPLETED
    db_session.expire_all()
    stored = db_session.query(Task).filter(Task.id == task.id).one()
    assert stored.status == TaskStatus.COMPLETED
    assert stored.output == "here is your answer"
    assert stored.runner_id is None


@pytest.mark.asyncio
async def test_runner_finalize_leaves_pending_when_deferred_to_ttl_recovery(
    db_session,
) -> None:
    """When settlement is deferred to TTL recovery (pool timeout), the delivery
    row must stay ``pending`` — the turn may be re-run by the recovery path."""
    user, task, payload, lease = _finalize_turn_fixture(
        db_session, turn_id="turn-e2e-defer"
    )

    async def pool_timeout_execute(*args, **kwargs):
        raise SQLAlchemyTimeoutError("setup/run pool exhausted")

    with _finalize_runner_patches(lease, execute=pool_timeout_execute):
        await _spawn_finalize_runner(task, user, payload)

    assert _delivery_status(db_session, "turn-e2e-defer") == DELIVERY_PENDING


@pytest.mark.asyncio
async def test_runner_finalize_leaves_pending_when_settlement_raises(
    db_session,
) -> None:
    """If lease settlement itself raises, the lease is retained for TTL recovery
    and the turn may re-run — so the delivery row must stay ``pending`` even
    though the run completed. Guards the settle-failure branch, which does not
    set ``defer_settlement_to_ttl_recovery``."""
    user, task, payload, lease = _finalize_turn_fixture(
        db_session, turn_id="turn-e2e-settle-raise"
    )

    with _finalize_runner_patches(
        lease, settle=MagicMock(side_effect=RuntimeError("settle boom"))
    ):
        await _spawn_finalize_runner(task, user, payload)

    assert _delivery_status(db_session, "turn-e2e-settle-raise") == DELIVERY_PENDING


@pytest.mark.asyncio
async def test_runner_finalize_leaves_pending_when_settle_is_fenced_out(
    db_session,
) -> None:
    """A fenced/superseded settlement must not close the delivery row.

    ``settle_task_lease_isolated`` returns ``False`` *without raising* when the
    lease row is gone or now owned by a different run. This stale coroutine is
    no longer authoritative for the turn, so reconciliation must be skipped and
    the ``pending`` row left to the authoritative owner."""
    user, task, payload, lease = _finalize_turn_fixture(
        db_session, turn_id="turn-e2e-fenced"
    )

    with _finalize_runner_patches(lease, settle=MagicMock(return_value=False)):
        await _spawn_finalize_runner(task, user, payload)

    assert _delivery_status(db_session, "turn-e2e-fenced") == DELIVERY_PENDING


@pytest.mark.asyncio
async def test_runner_finalize_skips_reconcile_on_settlement_ready_short_circuit(
    db_session,
) -> None:
    """Pre-execution SETTLEMENT_READY must not close the delivery row.

    A delayed run whose pre-acquired lease validates as SETTLEMENT_READY
    returns before the turn body executes. Settlement still runs (and here
    genuinely settles the lease), but the turn never ran, so the delivery row
    must not be reconciled to ``completed``."""
    user, task, payload, lease = _finalize_turn_fixture(
        db_session, turn_id="turn-e2e-settle-ready"
    )

    with _finalize_runner_patches(
        lease,
        validate=MagicMock(return_value=TaskLeaseRefreshState.SETTLEMENT_READY),
    ):
        await _spawn_finalize_runner(
            task, user, payload, task_lease=lease, run_id="run-a"
        )

    assert _delivery_status(db_session, "turn-e2e-settle-ready") == DELIVERY_PENDING


@pytest.mark.asyncio
async def test_runner_finalize_marks_dispatched_for_cancelled_turn(
    db_session,
) -> None:
    """A turn cancelled during execution closes its delivery row to
    ``dispatched``, not ``failed`` — execution had begun, so the message may
    already have been consumed and a fresh-id retry could double-execute it."""
    user, task, payload, lease = _finalize_turn_fixture(
        db_session, turn_id="turn-e2e-cancelled"
    )

    with _finalize_runner_patches(
        lease, execute=AsyncMock(side_effect=asyncio.CancelledError())
    ):
        bg_task = _spawn_finalize_runner(task, user, payload)
        with pytest.raises(asyncio.CancelledError):
            await bg_task

    assert _delivery_status(db_session, "turn-e2e-cancelled") == DELIVERY_DISPATCHED
    db_session.expire_all()
    stored = db_session.query(Task).filter(Task.id == task.id).one()
    assert stored.status == TaskStatus.FAILED


@pytest.mark.asyncio
async def test_runner_finalize_marks_dispatched_when_run_fails_after_start(
    db_session,
) -> None:
    """The double-execution hazard: a run that failed *after* execution began
    must close the orphaned ``pending`` row as ``dispatched`` (probe answers
    MATCHES, no retry), never ``failed`` (probe would invite a fresh-id resend
    of a message that already ran). This also pins the symmetry with the
    post-schedule projection: whichever best-effort write lands, the row
    converges on ``dispatched``."""
    user, task, payload, lease = _finalize_turn_fixture(
        db_session, turn_id="turn-e2e-late-fail"
    )

    with _finalize_runner_patches(
        lease,
        execute=AsyncMock(side_effect=RuntimeError("late boom after side effects")),
    ):
        await _spawn_finalize_runner(task, user, payload)

    assert _delivery_status(db_session, "turn-e2e-late-fail") == DELIVERY_DISPATCHED
    db_session.expire_all()
    stored = db_session.query(Task).filter(Task.id == task.id).one()
    assert stored.status == TaskStatus.FAILED


@pytest.mark.asyncio
async def test_runner_finalize_marks_failed_when_setup_fails_before_execute(
    db_session,
) -> None:
    """A failure before ``execute_task_background`` is ever invoked is positive
    evidence the message was never consumed, so the row closes as ``failed``
    and a fresh-id retry is safe."""
    user, task, payload, lease = _finalize_turn_fixture(
        db_session, turn_id="turn-e2e-pre-exec"
    )

    with _finalize_runner_patches(
        lease,
        snapshot=MagicMock(side_effect=RuntimeError("snapshot load exploded")),
    ):
        await _spawn_finalize_runner(task, user, payload)

    assert _delivery_status(db_session, "turn-e2e-pre-exec") == DELIVERY_FAILED
    db_session.expire_all()
    stored = db_session.query(Task).filter(Task.id == task.id).one()
    assert stored.status == TaskStatus.FAILED


@pytest.mark.asyncio
@pytest.mark.parametrize("shared", [False, True])
@pytest.mark.parametrize(
    ("initial_delivery", "expected_delivery"),
    [
        (DELIVERY_PENDING, DELIVERY_FAILED),
        (DELIVERY_DISPATCHED, DELIVERY_DISPATCHED),
    ],
)
async def test_runner_finalize_marks_failed_when_turn_files_no_longer_bind(
    db_session, monkeypatch, shared, initial_delivery, expected_delivery
) -> None:
    """A turn whose files no longer resolve when the run starts fails before
    ``execute_task_background`` is invoked, so an orphaned ``pending`` row
    closes as ``failed``. A row the post-schedule projection already marked
    ``dispatched`` stays ``dispatched``: delivery never regresses."""
    user, task, _, lease = _finalize_turn_fixture(
        db_session, turn_id="turn-e2e-unbindable"
    )
    payload = TaskTurnPayload(
        "hello there", file_ids=("fid-gone",), turn_id="turn-e2e-unbindable"
    )
    if initial_delivery == DELIVERY_DISPATCHED:
        mark_user_message_delivery(
            db_session,
            task_id=int(task.id),
            turn_id="turn-e2e-unbindable",
            status=DELIVERY_DISPATCHED,
        )
        db_session.commit()
    monkeypatch.setattr(
        task_orchestrator_module, "get_shared_task_execution_enabled", lambda: shared
    )
    execute = AsyncMock()

    with _finalize_runner_patches(lease, execute=execute):
        await _spawn_finalize_runner(task, user, payload)

    execute.assert_not_called()
    assert _delivery_status(db_session, "turn-e2e-unbindable") == expected_delivery
    db_session.expire_all()
    stored = db_session.query(Task).filter(Task.id == task.id).one()
    assert stored.status == TaskStatus.FAILED
    assert "Files are no longer bindable: fid-gone" in str(stored.error_message)


@pytest.mark.asyncio
async def test_runner_finalize_preserves_cancellation_from_reconcile(
    db_session,
) -> None:
    """A ``CancelledError`` surfacing from the reconcile call site is recorded
    as cleanup cancellation and re-raised after connector cleanup; the delivery
    row stays ``pending``."""
    user, task, payload, lease = _finalize_turn_fixture(
        db_session, turn_id="turn-e2e-cancel-reconcile"
    )

    with _finalize_runner_patches(
        lease,
        mark_delivery=MagicMock(side_effect=asyncio.CancelledError()),
    ):
        bg_task = _spawn_finalize_runner(task, user, payload)
        with pytest.raises(asyncio.CancelledError):
            await bg_task

    assert _delivery_status(db_session, "turn-e2e-cancel-reconcile") == DELIVERY_PENDING
    # The lease settlement itself committed before the reconcile cancellation.
    db_session.expire_all()
    stored = db_session.query(Task).filter(Task.id == task.id).one()
    assert stored.runner_id is None


def _shutdown_handoff_fixture(
    db_session,
    monkeypatch,
    *,
    turn_id: str,
    source: str = "internal",
    switch: str | None = "true",
    host_shutting_down: bool = True,
):
    """A RUNNING turn whose run the next await cancels, as a shutdown would."""
    if switch is None:
        monkeypatch.delenv("XAGENT_SHUTDOWN_HANDOFF_ENABLED", raising=False)
    else:
        monkeypatch.setenv("XAGENT_SHUTDOWN_HANDOFF_ENABLED", switch)
    monkeypatch.setattr(
        task_orchestrator_module,
        "_host_shutdown_in_progress",
        lambda: host_shutting_down,
    )
    user, task, payload, lease = _finalize_turn_fixture(db_session, turn_id=turn_id)
    task.source = source
    db_session.commit()
    return user, task, payload, lease


async def _run_cancelled_turn(task, user, payload, lease, *, user_cancel=False):
    from xagent.web.services.task_execution import background_task_manager

    with (
        _finalize_runner_patches(
            lease, execute=AsyncMock(side_effect=asyncio.CancelledError())
        ),
        patch.object(
            background_task_manager,
            "cancel_was_requested",
            return_value=user_cancel,
        ),
    ):
        bg_task = _spawn_finalize_runner(task, user, payload)
        with pytest.raises(asyncio.CancelledError):
            await bg_task


def _stored_task(db_session, task_id: int) -> Task:
    db_session.expire_all()
    return db_session.query(Task).filter(Task.id == task_id).one()


def _assert_lease_retained(db_session, task: Task, turn_id: str) -> None:
    stored = _stored_task(db_session, int(task.id))
    assert stored.status == TaskStatus.RUNNING
    assert stored.runner_id == "test-runner"
    assert stored.lease_attempt_id == "test-attempt"
    assert stored.run_id == "run-a"
    assert stored.error_message is None
    # Lease recovery closes the orphaned delivery row, as for any deferral.
    assert _delivery_status(db_session, turn_id) == DELIVERY_PENDING


def _assert_settled_cancelled(
    db_session, task: Task, *, error_message: str = "task execution cancelled"
) -> None:
    stored = _stored_task(db_session, int(task.id))
    assert stored.status == TaskStatus.FAILED
    assert stored.runner_id is None
    assert stored.error_message == error_message


@pytest.mark.asyncio
async def test_shutdown_cancel_hands_eligible_run_to_lease_recovery(
    db_session, monkeypatch
) -> None:
    """A host shutdown leaves an auto-resumable run RUNNING under its exact
    lease, so lease recovery pauses it and automatic resume continues it."""
    user, task, payload, lease = _shutdown_handoff_fixture(
        db_session, monkeypatch, turn_id="turn-shutdown-handoff"
    )

    await _run_cancelled_turn(task, user, payload, lease)

    _assert_lease_retained(db_session, task, "turn-shutdown-handoff")


@pytest.mark.asyncio
async def test_user_cancel_during_shutdown_still_settles(
    db_session, monkeypatch
) -> None:
    user, task, payload, lease = _shutdown_handoff_fixture(
        db_session, monkeypatch, turn_id="turn-shutdown-user-cancel"
    )

    await _run_cancelled_turn(task, user, payload, lease, user_cancel=True)

    _assert_settled_cancelled(db_session, task)


@pytest.mark.asyncio
@pytest.mark.parametrize("source", ["sdk", "a2a", "widget", "shared_link"])
async def test_shutdown_cancel_settles_sources_auto_recovery_excludes(
    db_session, monkeypatch, source
) -> None:
    """Callers that own their stop protocol, and SDK callers that only read
    the status (automatic recovery would resume an SDK task after a crash),
    still get a terminal state now."""
    user, task, payload, lease = _shutdown_handoff_fixture(
        db_session,
        monkeypatch,
        turn_id=f"turn-shutdown-{source}",
        source=source,
    )

    await _run_cancelled_turn(task, user, payload, lease)

    _assert_settled_cancelled(db_session, task)


@pytest.mark.asyncio
async def test_shutdown_cancel_settles_external_task_with_visitor_message(
    db_session, monkeypatch
) -> None:
    user, task, payload, lease = _shutdown_handoff_fixture(
        db_session,
        monkeypatch,
        turn_id="turn-shutdown-external",
        source=EXTERNAL_TASK_SOURCE,
    )

    await _run_cancelled_turn(task, user, payload, lease)

    _assert_settled_cancelled(
        db_session, task, error_message=EXTERNAL_TURN_INTERRUPTED_MESSAGE
    )


@pytest.mark.asyncio
async def test_shutdown_cancel_settles_channel_task(db_session, monkeypatch) -> None:
    """Channel tasks keep failing at shutdown, even where automatic recovery
    would resume them (shared execution)."""
    monkeypatch.setattr(
        "xagent.web.services.task_auto_recovery.get_shared_task_execution_enabled",
        lambda: True,
    )
    user, task, payload, lease = _shutdown_handoff_fixture(
        db_session, monkeypatch, turn_id="turn-shutdown-channel"
    )
    channel = UserChannel(
        user_id=user.id, channel_type="slack", channel_name="ops", config={}
    )
    db_session.add(channel)
    db_session.flush()
    task.channel_id = int(channel.id)
    db_session.commit()

    await _run_cancelled_turn(task, user, payload, lease)

    _assert_settled_cancelled(db_session, task)


@pytest.mark.asyncio
async def test_shutdown_cancel_settles_preview_task(db_session, monkeypatch) -> None:
    user, task, payload, lease = _shutdown_handoff_fixture(
        db_session, monkeypatch, turn_id="turn-shutdown-preview"
    )
    task.is_visible = False
    db_session.commit()

    await _run_cancelled_turn(task, user, payload, lease)

    _assert_settled_cancelled(db_session, task)


@pytest.mark.asyncio
async def test_shutdown_cancel_does_not_retain_a_row_another_writer_settled(
    db_session, monkeypatch
) -> None:
    """A row that already left RUNNING (a pause that landed while the host was
    going down) is not left under the cancelled run's lease."""
    user, task, payload, lease = _shutdown_handoff_fixture(
        db_session, monkeypatch, turn_id="turn-shutdown-already-paused"
    )
    task.status = TaskStatus.PAUSED
    db_session.commit()

    await _run_cancelled_turn(task, user, payload, lease)

    stored = _stored_task(db_session, int(task.id))
    assert stored.status == TaskStatus.PAUSED
    assert stored.runner_id is None
    assert stored.lease_attempt_id is None


@pytest.mark.asyncio
@pytest.mark.parametrize("switch", [None, "false"])
async def test_shutdown_cancel_settles_when_switch_off(
    db_session, monkeypatch, switch
) -> None:
    user, task, payload, lease = _shutdown_handoff_fixture(
        db_session,
        monkeypatch,
        turn_id="turn-shutdown-switch-off",
        switch=switch,
    )

    await _run_cancelled_turn(task, user, payload, lease)

    _assert_settled_cancelled(db_session, task)


@pytest.mark.asyncio
async def test_shutdown_cancel_settles_when_auto_resume_off(
    db_session, monkeypatch
) -> None:
    """Nothing would resume the paused run, so it fails as before."""
    monkeypatch.setenv("XAGENT_TASK_AUTO_RESUME_ENABLED", "false")
    user, task, payload, lease = _shutdown_handoff_fixture(
        db_session, monkeypatch, turn_id="turn-shutdown-auto-resume-off"
    )

    await _run_cancelled_turn(task, user, payload, lease)

    _assert_settled_cancelled(db_session, task)


@pytest.mark.asyncio
async def test_cancel_outside_shutdown_settles_with_switch_on(
    db_session, monkeypatch
) -> None:
    user, task, payload, lease = _shutdown_handoff_fixture(
        db_session,
        monkeypatch,
        turn_id="turn-cancel-no-shutdown",
        host_shutting_down=False,
    )

    await _run_cancelled_turn(task, user, payload, lease)

    _assert_settled_cancelled(db_session, task)


@pytest.mark.asyncio
async def test_shutdown_handoff_check_failure_settles(db_session, monkeypatch) -> None:
    user, task, payload, lease = _shutdown_handoff_fixture(
        db_session, monkeypatch, turn_id="turn-shutdown-check-fails"
    )
    monkeypatch.setattr(
        task_orchestrator_module,
        "_shutdown_handoff_eligible_sync",
        MagicMock(side_effect=RuntimeError("database unavailable")),
    )

    await _run_cancelled_turn(task, user, payload, lease)

    _assert_settled_cancelled(db_session, task)


@pytest.mark.asyncio
async def test_host_shutdown_in_progress_follows_both_shutdown_paths(
    monkeypatch,
) -> None:
    """Shared coordination cancels runs while closing its coordinators, before
    the background task manager's own shutdown starts."""
    from types import SimpleNamespace

    from xagent.web.services import task_coordinator_runtime
    from xagent.web.services.task_execution import background_task_manager

    loop = asyncio.get_running_loop()
    monkeypatch.setattr(background_task_manager, "_shutting_down", False)
    monkeypatch.setattr(task_coordinator_runtime, "_registry", None)
    assert task_orchestrator_module._host_shutdown_in_progress() is False

    monkeypatch.setattr(
        task_coordinator_runtime,
        "_registry",
        SimpleNamespace(_close_task=None, loop=loop),
    )
    assert task_orchestrator_module._host_shutdown_in_progress() is False

    monkeypatch.setattr(
        task_coordinator_runtime,
        "_registry",
        SimpleNamespace(_close_task=object(), loop=loop),
    )
    assert task_orchestrator_module._host_shutdown_in_progress() is True

    monkeypatch.setattr(task_coordinator_runtime, "_registry", None)
    monkeypatch.setattr(background_task_manager, "_shutting_down", True)
    assert task_orchestrator_module._host_shutdown_in_progress() is True


@pytest.mark.parametrize(
    "seeded_status, settlement_error, execution_started",
    [
        (DELIVERY_COMPLETED, None, True),
        (DELIVERY_COMPLETED, "boom", True),
        (DELIVERY_COMPLETED, "boom", False),
        (DELIVERY_FAILED, None, True),
    ],
)
def test_reconcile_finalized_delivery_noop_on_already_terminal_row(
    db_session,
    seeded_status,
    settlement_error,
    execution_started,
) -> None:
    """An already-terminal delivery row is never rewritten by reconciliation.

    ``completed`` and ``failed`` are terminal in the monotonic state machine;
    a later reconcile — idempotent or conflicting, under any target — must
    leave them as-is."""
    user = _create_user(db_session)
    task = _create_task(db_session, int(user.id), status=TaskStatus.RUNNING)
    _claim_pending_delivery(
        db_session, task_id=int(task.id), user_id=int(user.id), turn_id="turn-term"
    )
    transition = mark_user_message_delivery(
        db_session, task_id=int(task.id), turn_id="turn-term", status=seeded_status
    )
    assert transition.outcome == "updated"
    db_session.commit()
    db_session.expire_all()

    _reconcile_finalized_turn_delivery(
        task_id=int(task.id),
        turn_id="turn-term",
        settlement_error=settlement_error,
        execution_started=execution_started,
    )

    assert _delivery_status(db_session, "turn-term") == seeded_status


# ---------------------------------------------------------------------------
# Connector-runtime failures reach the client as a structured, wire-safe frame
# ---------------------------------------------------------------------------


CONNECTOR_RUNTIME_CODES = [
    "missing_runtime_context",
    "runtime_secret_unavailable",
    "scheduled_secret_unavailable",
]

# The reason a missing declared context key produces, used to populate the
# exception's own ``details`` below. The key half is a name the connector's
# owner chose; nothing under ``details`` reaches the terminal frame at all,
# so these fixtures exist to exercise the operator log, which still reads it.
WITHHELD_KEY_REASON = "missing_context.auth_token"
# An arbitrary reason value with no significance of its own -- it lives only
# in the exception's ``details``, which the frame never carries, so which
# string this is does not affect any assertion below.
PUBLIC_REASON = "not_provided"


@contextmanager
def _captured_terminal_broadcast(setup_or_run_error: BaseException, db_session):
    """Drive one owned run to failure and hand back both halves it produced.

    The branch under test writes two things: the broadcast frame the live
    client renders, and the durable settlement the transcript replays after a
    reload. Capturing only the frame would let the durable half be deleted
    with every test still green, so the settlement kwargs come back too.
    """

    from xagent.web.services.task_execution import background_task_manager

    user = _create_user(db_session)
    task = _create_task(db_session, user.id, status=TaskStatus.RUNNING)
    task_id = int(task.id)
    lease = TaskLease(
        task_id=task_id, runner_id="runner-a", run_id="run-a", attempt_id="test-attempt"
    )
    frames: list[dict] = []
    settlements: list[dict] = []

    async def broadcast(event, *_args, **_kwargs) -> None:
        frames.append(event)

    def settle(*_args, **kwargs) -> bool:
        settlements.append(kwargs)
        return True

    with (
        patch(
            "xagent.web.services.task_orchestrator.acquire_task_lease_isolated",
            return_value=lease,
        ),
        patch(
            "xagent.web.services.task_orchestrator.run_task_lease_heartbeat",
            new=AsyncMock(),
        ),
        patch(
            "xagent.web.services.task_orchestrator.load_task_setup_snapshot_sync",
            return_value=MagicMock(),
        ),
        patch.object(
            task_orchestrator_module,
            "resolve_execution_scope",
            return_value=None,
            create=True,
        ),
        patch(
            "xagent.web.services.task_execution.execute_task_background",
            new=AsyncMock(side_effect=setup_or_run_error),
        ),
        patch(
            "xagent.web.services.task_orchestrator.settle_task_lease_isolated",
            side_effect=settle,
        ),
        patch(
            "xagent.web.api.websocket.manager",
            MagicMock(broadcast_to_task=AsyncMock(side_effect=broadcast)),
        ),
        patch.object(background_task_manager, "register_task"),
        patch(
            "xagent.web.services.task_orchestrator._get_agent_manager",
            return_value=MagicMock(),
        ),
    ):
        yield task_id, frames, settlements


async def _run_failing_turn(task_id: int, user_id: int, source) -> None:
    await _schedule_bg(
        task_id=task_id,
        task_owner_user_id=user_id,
        task_source=source,
        payload=TaskTurnPayload("hello"),
        force_fresh=False,
        context=None,
    )


@pytest.mark.asyncio
@pytest.mark.parametrize("code", CONNECTOR_RUNTIME_CODES)
async def test_connector_runtime_failure_broadcasts_its_safe_message(
    db_session,
    code: str,
) -> None:
    """The curated sentence replaces the opaque task-failure fallback."""

    safe_message = f"Required connector runtime input is missing ({code})."
    error = ConnectorRuntimeError(
        code,
        safe_message,
        details={"reason": WITHHELD_KEY_REASON},
    )

    with _captured_terminal_broadcast(error, db_session) as (
        task_id,
        frames,
        settlements,
    ):
        task = db_session.query(Task).filter(Task.id == task_id).one()
        await _run_failing_turn(task_id, int(task.user_id), task.source)

    assert [frame["message"] for frame in frames] == [safe_message]
    assert frames[0]["error"] == safe_message
    assert frames[0]["code"] == code


@pytest.mark.asyncio
@pytest.mark.parametrize(
    "error",
    [
        ValueError("secret-token-xyz"),
        KeyError("secret-token-xyz"),
        RuntimeError("secret-token-xyz"),
    ],
    ids=["value-error", "key-error", "runtime-error"],
)
async def test_incidental_failure_still_redacts(
    db_session,
    error: BaseException,
) -> None:
    """Only the connector-runtime class earns the new branch."""

    with _captured_terminal_broadcast(error, db_session) as (
        task_id,
        frames,
        settlements,
    ):
        task = db_session.query(Task).filter(Task.id == task_id).one()
        await _run_failing_turn(task_id, int(task.user_id), task.source)

    assert len(frames) == 1
    assert frames[0]["message"] == CLIENT_SAFE_TASK_FAILURE
    assert frames[0]["error"] == CLIENT_SAFE_TASK_FAILURE
    assert "secret-token-xyz" not in json.dumps(frames[0])
    assert "code" not in frames[0]
    assert "details" not in frames[0]


@pytest.mark.asyncio
async def test_connector_runtime_frame_carries_code_only(db_session) -> None:
    """Whatever the raise site attached to ``details``, none of it reaches the wire."""

    error = ConnectorRuntimeError(
        "runtime_secret_unavailable",
        "Required runtime secret is unavailable.",
        details={
            "reason": PUBLIC_REASON,
            "internal_sql": "SELECT value FROM task_connector_runtime_contexts",
            "raw_value": "tenant-secret",
            "connector_ref": {"connector_type": "mcp", "connector_id": 7},
        },
    )

    with _captured_terminal_broadcast(error, db_session) as (
        task_id,
        frames,
        settlements,
    ):
        task = db_session.query(Task).filter(Task.id == task_id).one()
        await _run_failing_turn(task_id, int(task.user_id), task.source)

    assert set(frames[0]) == {
        "type",
        "message",
        "task_id",
        "task",
        "error",
        "timestamp",
        "code",
    }
    assert "details" not in frames[0]


@pytest.mark.asyncio
async def test_connector_runtime_frame_never_carries_connector_ref(
    db_session,
) -> None:
    """The frame's audience includes anonymous widget and share visitors.

    The three assertions are structural on purpose. An earlier form of this
    test also asserted the connector's numeric id was absent from the
    serialized frame, which goes red on any fixture where that id collides
    with the task id or a timestamp digit.
    """

    error = ConnectorRuntimeError(
        "runtime_secret_unavailable",
        "Required runtime secret is unavailable.",
        connector_ref=ConnectorRef(connector_type="mcp", connector_id=7),
        details={"reason": PUBLIC_REASON},
    )

    with _captured_terminal_broadcast(error, db_session) as (
        task_id,
        frames,
        settlements,
    ):
        task = db_session.query(Task).filter(Task.id == task_id).one()
        await _run_failing_turn(task_id, int(task.user_id), task.source)

    frame = frames[0]
    assert set(frame) == {
        "type",
        "message",
        "task_id",
        "task",
        "error",
        "timestamp",
        "code",
    }
    serialized = json.dumps(frame)
    assert "connector_ref" not in serialized
    assert "connector_id" not in serialized


@pytest.mark.asyncio
@pytest.mark.parametrize("code", CONNECTOR_RUNTIME_CODES)
async def test_connector_runtime_failure_logs_missing_key(
    db_session,
    caplog,
    code: str,
) -> None:
    """Operators read the raw details, connector identity included."""

    error = ConnectorRuntimeError(
        code,
        "Required connector runtime context is missing.",
        connector_ref=ConnectorRef(connector_type="mcp", connector_id=7),
        details={"reason": WITHHELD_KEY_REASON},
    )

    with caplog.at_level(logging.ERROR):
        with _captured_terminal_broadcast(error, db_session) as (
            task_id,
            frames,
            settlements,
        ):
            task = db_session.query(Task).filter(Task.id == task_id).one()
            await _run_failing_turn(task_id, int(task.user_id), task.source)

    structured = [
        record.getMessage()
        for record in caplog.records
        if record.levelno == logging.ERROR
        and "component=connector-runtime" in record.getMessage()
    ]
    assert len(structured) == 1
    assert f"code={code}" in structured[0]
    assert f"reason={WITHHELD_KEY_REASON}" in structured[0]
    assert "connector=" in structured[0]
    assert "'connector_id': 7" in structured[0]


@pytest.mark.asyncio
@pytest.mark.parametrize("code", CONNECTOR_RUNTIME_CODES)
async def test_connector_runtime_failure_persists_client_safe_history(
    db_session,
    code: str,
) -> None:
    """The durable half: what the transcript replays after a reload.

    The new branch writes three things -- the frame, the settlement error and
    the history message type. Without this test the whole
    ``client_history_message_type`` line could be deleted and every other test
    in this file would stay green, while a reloading user dropped back to the
    generic failure text the frame no longer shows.
    """

    safe_message = f"Required connector runtime input is missing ({code})."
    error = ConnectorRuntimeError(
        code,
        safe_message,
        details={"reason": WITHHELD_KEY_REASON},
    )

    with _captured_terminal_broadcast(error, db_session) as (
        task_id,
        frames,
        settlements,
    ):
        task = db_session.query(Task).filter(Task.id == task_id).one()
        await _run_failing_turn(task_id, int(task.user_id), task.source)

    assert len(settlements) == 1
    settled = settlements[0]
    assert settled["client_message_type"] == CLIENT_SAFE_FAILURE_MESSAGE_TYPE
    # The durable row and the frame's own message field are the same server
    # sentence. The live bubble is not that sentence: for a missing-value code
    # the client replaces it with its own localized wording (see the
    # "terminal error frames" suite in app-context-chat.test.tsx). What the two
    # views owe each other is the facts they carry, and the key name is in
    # neither -- the frame never carries a details object at all, so there is
    # no key name for the client to render.
    assert settled["client_error_message"] == safe_message
    assert settled["client_error_message"] == frames[0]["message"]
    assert "details" not in frames[0]
    assert "auth_token" not in json.dumps(frames[0])
    # The durable error keeps the code prefix operators grep for, and never
    # the "setup/run error: <ExceptionType>" shape the else branch produces.
    assert settled["error_message"] == f"{code}: {safe_message}"
    assert "setup/run error" not in settled["error_message"]


@pytest.mark.asyncio
async def test_incidental_failure_persists_the_generic_history_type(
    db_session,
) -> None:
    """The counterpart: an incidental failure keeps the untrusted settlement."""

    error = RuntimeError("secret-token-xyz")

    with _captured_terminal_broadcast(error, db_session) as (
        task_id,
        frames,
        settlements,
    ):
        task = db_session.query(Task).filter(Task.id == task_id).one()
        await _run_failing_turn(task_id, int(task.user_id), task.source)

    assert len(settlements) == 1
    settled = settlements[0]
    assert settled["client_message_type"] == TASK_FAILURE_MESSAGE_TYPE
    assert settled["client_error_message"] == CLIENT_SAFE_TASK_FAILURE
    assert "secret-token-xyz" not in settled["client_error_message"]


@pytest.mark.asyncio
async def test_trigger_metrics_cancelled_before_runner_starts(monkeypatch) -> None:
    from xagent.core.utils import setup_metrics
    from xagent.web.services.task_execution import background_task_manager

    counters = setup_metrics.SetupMetrics()
    monkeypatch.setattr(setup_metrics, "trigger_execution", counters)
    with patch.object(background_task_manager, "register_task"):
        background = _schedule_bg(
            task_id=1,
            task_owner_user_id=1,
            task_source="trigger",
            payload=TaskTurnPayload("hello"),
            force_fresh=False,
            context=None,
        )
        assert counters.active == 1
        background.cancel()
        with pytest.raises(asyncio.CancelledError):
            await background
    assert counters.active == 0
    assert counters.completed == 1
    assert counters.cancelled == 1
    assert counters.failed == 0


@pytest.mark.asyncio
async def test_leased_auto_failure_preserves_client_classification(db_session) -> None:
    error = AutoModelUnavailableError("private model binding details")
    with _captured_terminal_broadcast(error, db_session) as (
        task_id,
        frames,
        settlements,
    ):
        task = db_session.query(Task).filter(Task.id == task_id).one()
        with patch(
            "xagent.web.services.task_execution.execute_task_background",
            new=AsyncMock(side_effect=error),
        ) as execute:
            await _run_failing_turn(task_id, int(task.user_id), task.source)
        execute.assert_awaited_once()
        assert execute.await_args.kwargs["task_lease"] == TaskLease(
            task_id=task_id,
            runner_id="runner-a",
            run_id="run-a",
            attempt_id="test-attempt",
        )

    assert len(frames) == 1
    assert frames[0]["code"] == "auto_model_unavailable"
    assert frames[0]["message"] == CLIENT_SAFE_AUTO_MODEL_UNAVAILABLE
    assert len(settlements) == 1
    assert settlements[0]["client_message_type"] == CLIENT_SAFE_FAILURE_MESSAGE_TYPE
    assert settlements[0]["client_error_message"] == CLIENT_SAFE_AUTO_MODEL_UNAVAILABLE
    assert settlements[0]["error_message"] == CLIENT_SAFE_AUTO_MODEL_UNAVAILABLE
    assert "private model binding details" not in json.dumps(frames)


@pytest_asyncio.fixture
async def cancellation_trace_runtime(db_session, monkeypatch, async_trace):
    from xagent.web.services import trace_handlers
    from xagent.web.services.trace_database import TraceDatabaseRuntime

    runtime = TraceDatabaseRuntime(
        db_session.get_bind(), use_async=async_trace, limit=1
    )
    monkeypatch.setattr(trace_handlers, "get_trace_database_runtime", lambda: runtime)
    try:
        yield runtime
    finally:
        await runtime.close()


@pytest.mark.asyncio
@pytest.mark.parametrize(
    ("kind", "async_trace"),
    [
        ("checkpoint", False),
        ("trace", False),
        ("outbound", False),
        ("checkpoint", True),
        ("trace", True),
    ],
)
@pytest.mark.parametrize("write_fails", [False, True])
async def test_cancelled_runner_drains_persistence_before_settlement(
    db_session, monkeypatch, kind, write_fails, async_trace, cancellation_trace_runtime
):
    # Exercise settlement through both real file-SQLite driver paths. The
    # async commit barrier yields through run_sync, never blocks the loop.
    from functools import partial

    from sqlalchemy.ext.asyncio import AsyncSession
    from sqlalchemy.util.concurrency import await_only

    from xagent.core.agent.checkpoint import CHECKPOINT_EVENT_TYPE
    from xagent.core.agent.trace import TraceEvent as CoreTraceEvent
    from xagent.web.api import websocket
    from xagent.web.services import task_execution, trace_handlers
    from xagent.web.services.task_lease_service import bind_task_lease_context

    turn_id = f"cancel-persist-{kind}-{write_fails}"
    user, task, payload, lease = _finalize_turn_fixture(db_session, turn_id=turn_id)
    started, release, closed, rolled_back = Event(), Event(), Event(), Event()
    async_release = asyncio.Event()
    heartbeat_stopped = asyncio.Event()
    settlement_started = Event()
    error = RuntimeError("persistence failed during cancellation")

    class WriterSession(Session):
        def commit(self):
            self.flush()
            started.set()
            if async_trace:
                await_only(asyncio.wait_for(async_release.wait(), 8))
            else:
                assert release.wait(8)
            if write_fails:
                raise error
            return super().commit()

        def rollback(self):
            result = super().rollback()
            rolled_back.set()
            return result

        def close(self):
            super().close()
            closed.set()

    factory = sessionmaker(db_session.get_bind(), class_=WriterSession)
    if async_trace:
        from xagent.web.services import trace_database

        assert cancellation_trace_runtime.engine is not None
        monkeypatch.setattr(
            trace_database,
            "AsyncSession",
            partial(AsyncSession, sync_session_class=WriterSession),
        )
    monkeypatch.setattr(trace_handlers, "get_db", lambda: iter([factory()]))
    monkeypatch.setattr(task_execution, "get_db", lambda: iter([factory()]))
    broadcast = AsyncMock()
    monkeypatch.setattr(websocket.manager, "broadcast_to_task", broadcast)

    async def execute(*args, **kwargs):
        with bind_task_lease_context(lease):
            if kind == "outbound":
                await task_execution.make_agent_outbound_handler(task.id)(
                    {"message": "progress", "event_id": turn_id}
                )
            else:
                event = CoreTraceEvent(
                    CHECKPOINT_EVENT_TYPE,
                    task_id=str(task.id),
                    step_id="step",
                    data={
                        "checkpoint_type": CHECKPOINT_TYPE,
                        "execution_id": "cancel-test",
                        "snapshot": {},
                    }
                    if kind == "checkpoint"
                    else {"message": "progress"},
                    require_persisted=kind == "checkpoint",
                )
                await trace_handlers.DatabaseTraceHandler(task.id)._save_to_database(
                    event
                )

    async def heartbeat(_lease, stop_event):
        await stop_event.wait()
        heartbeat_stopped.set()
        return TaskLeaseHeartbeatOutcome()

    settle = task_orchestrator_module.settle_task_lease_isolated

    def checked_settle(*args, **kwargs):
        settlement_started.set()
        assert closed.is_set()
        assert heartbeat_stopped.is_set()
        assert rolled_back.is_set() == write_fails
        return settle(*args, **kwargs)

    with _finalize_runner_patches(
        lease, execute=execute, settle=checked_settle, heartbeat=heartbeat
    ):
        bg_task = _spawn_finalize_runner(task, user, payload)
        try:
            async with asyncio.timeout(DB_PROGRESS_TIMEOUT):
                while not started.is_set():
                    await asyncio.sleep(0.001)
            for _ in range(2):
                bg_task.cancel()
                # Give cancellation a complete event-loop turn to reach the
                # persistence boundary; the worker stays behind its barrier.
                await asyncio.sleep(0)
                await asyncio.sleep(0)
                assert not bg_task.done()
                assert not closed.is_set()
                assert not settlement_started.is_set()
                assert not heartbeat_stopped.is_set()
        finally:
            release.set()
            async_release.set()
        with pytest.raises(asyncio.CancelledError):
            await asyncio.wait_for(bg_task, DB_PROGRESS_TIMEOUT)

    assert settlement_started.is_set()
    db_session.expire_all()
    stored = db_session.get(Task, task.id)
    assert stored.status == TaskStatus.FAILED
    assert bool(stored.last_checkpoint_event_id) == (
        kind == "checkpoint" and not write_fails
    )
    assert _delivery_status(db_session, turn_id) == DELIVERY_DISPATCHED
    if kind == "outbound":
        assert not any(
            call.args[0].get("event_id") == turn_id
            for call in broadcast.await_args_list
        )


@pytest.mark.parametrize("commit", [False, True])
def test_acceptance_snapshot_stages_no_execution_lease(db_session, commit):
    from xagent.web.services.task_orchestrator import _accept_turn_no_commit

    user = _create_user(db_session)
    task = _create_task(db_session, int(user.id))
    task_id = int(task.id)
    accepted = _accept_turn_no_commit(
        db_session,
        task_id,
        int(user.id),
        payload=TaskTurnPayload(transcript_message="Accepted", turn_id="accept-only"),
        kind=TurnKind.CREATE,
    )
    db_session.refresh(task)
    assert accepted.run_id == task.run_id
    assert accepted.status == TaskStatus.RUNNING
    assert task.runner_id is None
    assert task.lease_attempt_id is None
    assert task.lease_expires_at is None
    assert task.last_heartbeat_at is None
    if commit:
        db_session.commit()
    else:
        db_session.rollback()
    db_session.expire_all()
    task = db_session.get(Task, task_id)
    assert task.status == (TaskStatus.RUNNING if commit else TaskStatus.PENDING)
    assert db_session.query(TaskChatMessage).filter_by(task_id=task_id).count() == int(
        commit
    )


@pytest.mark.asyncio
async def test_unknown_settlement_flushes_workforce_projection(db_session):
    from xagent.web.services.task_execution import _acquire_resume_task_lease
    from xagent.web.services.task_orchestrator import pause_unknown_task_lease

    user = _create_user(db_session)
    manager = Agent(user_id=user.id, name="unknown projection manager")
    db_session.add(manager)
    db_session.flush()
    workforce = Workforce(
        owner_user_id=user.id,
        scope_type="user",
        scope_id=str(user.id),
        name="unknown workforce",
        manager_agent_id=manager.id,
        status="active",
    )
    db_session.add(workforce)
    db_session.flush()
    task = _create_task(db_session, user.id, status=TaskStatus.PAUSED)
    run = WorkforceRun(
        workforce_id=workforce.id,
        task_id=task.id,
        user_id=user.id,
        status="paused",
        snapshot={"version": 1},
    )
    db_session.add(run)
    db_session.flush()
    task.agent_config = {"workforce_run_id": int(run.id)}
    db_session.commit()
    lease = _acquire_resume_task_lease(int(task.id), int(user.id), None)
    assert lease is not None
    assert await pause_unknown_task_lease(lease)
    db_session.expire_all()
    assert db_session.get(Task, task.id).status == TaskStatus.PAUSED
    assert db_session.get(WorkforceRun, run.id).status == "paused"


# --- unsettled evidence for a pooled release ------------------------------------
#
# ``_schedule_bg`` either settles its run lease or leaves it to TTL recovery
# (deferred settlement, or a settlement that raised). Re-run unchanged, the
# scenarios above must grow the process hold's sticky ``unsettled_total`` by
# one exactly for a retained lease. A fenced-out settlement (``False``) is not
# counted: by contract it also covers "already terminal, reconciled and
# released", so it is no evidence of a retained lease.

UNSETTLED_SCENARIOS = [
    (test_runner_finalize_reconciles_orphaned_pending_delivery, 0),
    (test_runner_finalize_reconciles_alongside_a_genuinely_completed_turn, 0),
    (test_runner_finalize_leaves_pending_when_settle_is_fenced_out, 0),
    (test_schedule_bg_settles_owned_terminal_task_observed_by_heartbeat, 0),
    (test_runner_finalize_leaves_pending_when_deferred_to_ttl_recovery, 1),
    (test_runner_finalize_leaves_pending_when_settlement_raises, 1),
    (test_schedule_bg_heartbeat_pool_timeout_skips_settlement, 1),
    (test_schedule_bg_pool_timeout_defers_settlement_to_lease_recovery, 1),
]


@pytest.mark.asyncio
@pytest.mark.parametrize(
    ("scenario", "expected"),
    UNSETTLED_SCENARIOS,
    ids=[scenario.__name__ for scenario, _ in UNSETTLED_SCENARIOS],
)
async def test_schedule_bg_records_unsettled_only_when_the_lease_is_retained(
    request: pytest.FixtureRequest, scenario, expected: int
) -> None:
    import inspect

    from xagent.web.services import release_hold as release_hold_module

    hold = release_hold_module.release_hold
    kwargs = {
        name: request.getfixturevalue(name)
        for name in inspect.signature(scenario).parameters
    }
    before = hold.unsettled_total
    await scenario(**kwargs)
    assert hold.unsettled_total - before == expected
    if expected:
        assert hold.snapshot()["unsettled"][-1]["kind"] == "task_run_lease"


@pytest.mark.asyncio
@pytest.mark.parametrize(
    ("settle_outcome", "expected"), [(True, 0), (False, 0), ("raises", 1)]
)
async def test_settlement_cancelled_again_records_its_own_outcome(
    db_session, settle_outcome, expected: int
) -> None:
    """A second cancel during settlement does not decide the record.

    The runner was cancelled and is settling; a further cancel arrives while
    the settlement transaction runs. The settlement is drained and the runner
    still ends cancelled, but only a settlement that itself raised leaves the
    lease to TTL recovery.
    """
    from xagent.web.services import release_hold as release_hold_module

    user, task, payload, lease = _finalize_turn_fixture(
        db_session, turn_id=f"cancel-settle-{settle_outcome}"
    )
    started = asyncio.Event()
    settling, proceed = Event(), Event()

    async def execute(*_args, **_kwargs) -> None:
        started.set()
        await asyncio.Event().wait()

    async def heartbeat(_lease, stop_event):
        await stop_event.wait()
        return TaskLeaseHeartbeatOutcome()

    def blocking_settle(*_args, **_kwargs) -> bool:
        settling.set()
        assert proceed.wait(DB_PROGRESS_TIMEOUT)
        if settle_outcome == "raises":
            raise RuntimeError("settlement commit failed")
        return bool(settle_outcome)

    hold = release_hold_module.release_hold
    before = hold.unsettled_total
    with _finalize_runner_patches(
        lease, execute=execute, settle=blocking_settle, heartbeat=heartbeat
    ):
        bg_task = _spawn_finalize_runner(task, user, payload)
        try:
            await asyncio.wait_for(started.wait(), DB_PROGRESS_TIMEOUT)
            bg_task.cancel()
            assert await asyncio.to_thread(settling.wait, DB_PROGRESS_TIMEOUT)
            bg_task.cancel()
            await asyncio.sleep(0)
        finally:
            proceed.set()
        with pytest.raises(asyncio.CancelledError):
            await asyncio.wait_for(bg_task, DB_PROGRESS_TIMEOUT)
    assert hold.unsettled_total - before == expected