Skip to content

Enterprise agent platform for building, deploying, and operating AI teammates.

75.2 KB
Raw Blame
"""Tests for ``AgentServiceManager.execute_task`` lease delegation."""

from __future__ import annotations

import asyncio
import threading
from datetime import datetime, timezone
from unittest.mock import AsyncMock, MagicMock, patch

import pytest
from sqlalchemy import create_engine
from sqlalchemy.exc import TimeoutError as SQLAlchemyTimeoutError
from sqlalchemy.orm import sessionmaker
from sqlalchemy.pool import QueuePool

from tests.shared.async_waits import DB_PROGRESS_TIMEOUT
from tests.web.pool_contention_shared import (
    GUARD_TIMEOUT,
    LOOP_LIVENESS_TICKS,
    gated_pool_checkout,
    wait_for_ticks,
)
from xagent.core.model.chat.token_context import TokenUsage
from xagent.web.models.agent import Agent, AgentStatus
from xagent.web.models.database import Base, get_db, get_engine, init_db
from xagent.web.models.task import Task, TaskStatus
from xagent.web.models.user import User
from xagent.web.models.workforce import Workforce, WorkforceRun
from xagent.web.services import task_lease_service
from xagent.web.services.agent_service_manager import (
    AgentServiceManager,
    _update_task_title_isolated,
)
from xagent.web.services.task_lease_service import (
    TaskLease,
    TaskLeaseHeartbeatOutcome,
    TaskLeaseLostError,
    current_task_lease,
)
from xagent.web.services.workforce_runtime import (
    sync_workforce_run_status,
    sync_workforce_run_status_for_task_id_isolated,
)
from xagent.web.tracking.task_tracker import _TaskTrackingSeed


@pytest.fixture()
def db_session(tmp_path):
    init_db(db_url=f"sqlite:///{tmp_path / 'execute_task_lease.db'}")
    db = next(get_db())
    try:
        yield db
    finally:
        db.close()
        Base.metadata.drop_all(bind=get_engine())


class _FakeAgentService:
    async def execute_task(self, **_kwargs):
        return {"success": True}

    def set_interrupt_checker(self, _checker):
        # execute_task_background sets the mid-run quota checker after tracking
        # starts and clears it on completion; the double must accept both.
        pass


def _create_single_connection_runtime_db(tmp_path, filename: str):
    engine = create_engine(
        f"sqlite:///{tmp_path / filename}",
        poolclass=QueuePool,
        pool_size=1,
        max_overflow=0,
        pool_timeout=0.05,
        connect_args={"check_same_thread": False},
    )
    User.__table__.create(bind=engine)
    Task.__table__.create(bind=engine)
    factory = sessionmaker(bind=engine, autocommit=False, autoflush=False)
    with factory() as setup_db:
        user = User(username=f"{filename}-user", password_hash="hash", is_admin=False)
        setup_db.add(user)
        setup_db.flush()
        task = Task(
            user_id=user.id,
            title="pool test",
            description="test",
            status=TaskStatus.RUNNING,
            execution_mode="auto",
        )
        setup_db.add(task)
        setup_db.commit()
        task_id = int(task.id)
    return engine, factory, task_id


@pytest.mark.asyncio
async def test_execute_task_binds_outer_lease_only_during_agent_execution() -> None:
    manager = AgentServiceManager()
    lease = TaskLease(
        task_id=42, runner_id="runner-a", run_id="run-a", attempt_id="test-attempt"
    )
    observed_leases: list[TaskLease | None] = []

    class LeaseObservingAgent(_FakeAgentService):
        async def execute_task(self, **_kwargs):
            observed_leases.append(current_task_lease())
            return {"success": True}

    with (
        patch.object(
            manager,
            "_acquire_sandbox_task",
            new=AsyncMock(return_value=None),
        ),
        patch.object(manager, "_release_sandbox_task", new=AsyncMock()),
        patch(
            "xagent.web.services.agent_service_manager.stop_task_lease_heartbeat",
            new=AsyncMock(),
        ),
    ):
        result = await manager.execute_task(
            agent_service=LeaseObservingAgent(),
            task="hello",
            manage_task_lease=False,
            task_lease=lease,
        )

    assert result["success"] is True
    assert observed_leases == [lease]
    assert current_task_lease() is None


class _MustNotRunAgent(_FakeAgentService):
    async def execute_task(self, **_kwargs):
        raise AssertionError("an unleased shared run must not reach the agent")


@pytest.mark.asyncio
@pytest.mark.parametrize(
    ("manage_task_lease", "task_id"),
    [
        # An outer orchestrator claims ownership but hands over no lease.
        (False, "42"),
        # Nothing to acquire a lease for.
        (True, None),
    ],
)
async def test_shared_execute_task_refuses_a_run_without_a_lease(
    manage_task_lease: bool, task_id: str | None
) -> None:
    manager = AgentServiceManager()
    acquire_sandbox = AsyncMock(return_value=None)

    with (
        patch(
            "xagent.web.services.agent_service_manager."
            "get_shared_task_execution_enabled",
            return_value=True,
        ),
        patch(
            "xagent.web.services.agent_service_manager.acquire_task_lease_isolated",
        ) as mock_acquire,
        patch.object(manager, "_acquire_sandbox_task", new=acquire_sandbox),
    ):
        with pytest.raises(RuntimeError, match="requires a task lease"):
            await manager.execute_task(
                agent_service=_MustNotRunAgent(),
                task="hello",
                task_id=task_id,
                manage_task_lease=manage_task_lease,
            )

    mock_acquire.assert_not_called()
    acquire_sandbox.assert_not_awaited()


@pytest.mark.asyncio
async def test_local_execute_task_still_runs_without_a_lease() -> None:
    manager = AgentServiceManager()

    with (
        patch(
            "xagent.web.services.agent_service_manager."
            "get_shared_task_execution_enabled",
            return_value=False,
        ),
        patch.object(
            manager, "_acquire_sandbox_task", new=AsyncMock(return_value=None)
        ),
        patch.object(manager, "_release_sandbox_task", new=AsyncMock()),
        patch(
            "xagent.web.services.agent_service_manager.stop_task_lease_heartbeat",
            new=AsyncMock(),
        ),
    ):
        result = await manager.execute_task(
            agent_service=_FakeAgentService(),
            task="hello",
            manage_task_lease=False,
        )

    assert result["success"] is True


@pytest.mark.asyncio
async def test_execute_task_tracks_usage_for_outer_owned_lease() -> None:
    manager = AgentServiceManager()
    lease = TaskLease(
        task_id=42, runner_id="runner-a", run_id="run-a", attempt_id="test-attempt"
    )
    tracker = MagicMock(
        start_tracking=AsyncMock(),
        complete_tracking=AsyncMock(),
        stop_periodic_updates=AsyncMock(),
        interrupt_reason_for_quota=AsyncMock(),
        quota_interrupt_reason=None,
    )

    with (
        patch(
            "xagent.web.services.agent_service_manager._load_task_run_gate_user_id_isolated",
            return_value=7,
        ),
        patch(
            "xagent.web.services.agent_service_manager._check_task_run_gate_on_event_loop",
            return_value=None,
        ),
        patch(
            "xagent.web.services.agent_service_manager.sync_workforce_run_status_for_task_id_isolated"
        ) as sync_workforce,
        patch(
            "xagent.web.tracking.task_tracker.TaskTracker",
            return_value=tracker,
        ) as tracker_factory,
        patch.object(
            manager,
            "_acquire_sandbox_task",
            new=AsyncMock(return_value=None),
        ),
        patch.object(manager, "_release_sandbox_task", new=AsyncMock()),
    ):
        result = await manager.execute_task(
            agent_service=_FakeAgentService(),
            task="hello",
            tracking_task_id="42",
            manage_task_lease=False,
            task_lease=lease,
        )

    assert result["success"] is True
    sync_workforce.assert_not_called()
    tracker_factory.assert_called_once_with(
        task_id=42,
        expected_run_id="run-a",
        expected_runner_id="runner-a",
        expected_attempt_id="test-attempt",
    )
    tracker.start_tracking.assert_awaited_once()
    tracker.complete_tracking.assert_awaited_once()


@pytest.mark.asyncio
async def test_execute_task_external_lease_loss_cancels_agent_execution() -> None:
    manager = AgentServiceManager()
    lease = TaskLease(
        task_id=42, runner_id="runner-a", run_id="run-a", attempt_id="test-attempt"
    )
    execution_started = asyncio.Event()
    execution_cancelled = asyncio.Event()

    class BlockingAgent(_FakeAgentService):
        async def execute_task(self, **_kwargs):
            execution_started.set()
            try:
                await asyncio.Event().wait()
            finally:
                execution_cancelled.set()

    async def lose_lease() -> TaskLeaseHeartbeatOutcome:
        await execution_started.wait()
        return TaskLeaseHeartbeatOutcome(lease_lost=True)

    heartbeat_task = asyncio.create_task(lose_lease())
    with (
        patch.object(
            manager,
            "_acquire_sandbox_task",
            new=AsyncMock(return_value=None),
        ),
        patch.object(manager, "_release_sandbox_task", new=AsyncMock()),
        patch(
            "xagent.web.services.agent_service_manager.stop_task_lease_heartbeat",
            new=AsyncMock(),
        ),
    ):
        with pytest.raises(TaskLeaseLostError):
            await manager.execute_task(
                agent_service=BlockingAgent(),
                task="hello",
                manage_task_lease=False,
                task_lease=lease,
                task_lease_heartbeat_task=heartbeat_task,
            )

    assert execution_cancelled.is_set()


@pytest.mark.asyncio
async def test_execute_task_managed_lease_loss_skips_usage_and_release() -> None:
    manager = AgentServiceManager()
    lease = TaskLease(
        task_id=42, runner_id="runner-a", run_id="run-a", attempt_id="test-attempt"
    )
    execution_started = asyncio.Event()
    execution_cancelled = asyncio.Event()
    tracker = MagicMock(
        start_tracking=AsyncMock(),
        complete_tracking=AsyncMock(),
        stop_periodic_updates=AsyncMock(),
        interrupt_reason_for_quota=AsyncMock(),
    )

    class BlockingAgent(_FakeAgentService):
        async def execute_task(self, **_kwargs):
            execution_started.set()
            try:
                await asyncio.Event().wait()
            finally:
                execution_cancelled.set()

    async def lose_lease(
        _lease: TaskLease,
        _stop_event: asyncio.Event,
    ) -> TaskLeaseHeartbeatOutcome:
        await execution_started.wait()
        return TaskLeaseHeartbeatOutcome(lease_lost=True)

    release = MagicMock(return_value=True)
    with (
        patch(
            "xagent.web.services.agent_service_manager._load_task_run_gate_user_id_isolated",
            return_value=None,
        ),
        patch(
            "xagent.web.services.agent_service_manager.acquire_task_lease_isolated",
            return_value=lease,
        ),
        patch(
            "xagent.web.services.agent_service_manager.run_task_lease_heartbeat",
            new=lose_lease,
        ),
        patch(
            "xagent.web.services.agent_service_manager._release_managed_task_lease_isolated",
            release,
        ),
        patch(
            "xagent.web.services.agent_service_manager.sync_workforce_run_status_for_task_id_isolated",
            return_value=False,
        ),
        patch(
            "xagent.web.tracking.task_tracker.TaskTracker",
            return_value=tracker,
        ),
        patch.object(
            manager,
            "_acquire_sandbox_task",
            new=AsyncMock(return_value=None),
        ),
        patch.object(manager, "_release_sandbox_task", new=AsyncMock()),
    ):
        with pytest.raises(TaskLeaseLostError):
            await manager.execute_task(
                agent_service=BlockingAgent(),
                task="hello",
                tracking_task_id="42",
                manage_task_lease=True,
            )

    assert execution_cancelled.is_set()
    tracker.complete_tracking.assert_not_awaited()
    tracker.stop_periodic_updates.assert_awaited_once()
    release.assert_not_called()


@pytest.mark.asyncio
async def test_execute_task_lease_loss_during_title_update_cancels_stale_write() -> (
    None
):
    manager = AgentServiceManager()
    lease = TaskLease(
        task_id=42, runner_id="runner-a", run_id="run-a", attempt_id="test-attempt"
    )
    title_started = asyncio.Event()
    title_cancelled = asyncio.Event()

    async def update_title(*_args, **_kwargs) -> bool:
        title_started.set()
        try:
            await asyncio.Event().wait()
        finally:
            title_cancelled.set()

    async def lose_lease() -> TaskLeaseHeartbeatOutcome:
        await title_started.wait()
        return TaskLeaseHeartbeatOutcome(lease_lost=True)

    heartbeat_task = asyncio.create_task(lose_lease())
    with (
        patch.object(
            manager,
            "_acquire_sandbox_task",
            new=AsyncMock(return_value=None),
        ),
        patch.object(manager, "_release_sandbox_task", new=AsyncMock()),
        patch(
            "xagent.web.services.agent_service_manager.update_task_title_from_agent",
            new=update_title,
        ),
        patch(
            "xagent.web.services.agent_service_manager.stop_task_lease_heartbeat",
            new=AsyncMock(),
        ),
    ):
        with pytest.raises(TaskLeaseLostError):
            await manager.execute_task(
                agent_service=_FakeAgentService(),
                task="hello",
                task_id="42",
                manage_task_lease=False,
                task_lease=lease,
                task_lease_heartbeat_task=heartbeat_task,
            )

    assert title_cancelled.is_set()


@pytest.mark.asyncio
async def test_execute_task_quota_pool_timeout_stops_pre_run_checkouts(
    tmp_path,
) -> None:
    """A run-gate context timeout stops later workforce/tracker checkouts."""
    engine, factory, task_id = _create_single_connection_runtime_db(
        tmp_path,
        "quota-pool-timeout.db",
    )
    held_connection = engine.connect()
    manager = AgentServiceManager()

    try:
        with (
            patch("xagent.web.models.database._SessionLocal", factory),
            patch(
                "xagent.web.services.agent_service_manager.sync_workforce_run_status_for_task_id_isolated",
            ) as workforce_sync,
            patch(
                "xagent.web.tracking.task_tracker.TaskTracker",
            ) as tracker_factory,
        ):
            with pytest.raises(SQLAlchemyTimeoutError, match="QueuePool limit"):
                await manager.execute_task(
                    agent_service=_FakeAgentService(),
                    task="hello",
                    tracking_task_id=str(task_id),
                    manage_task_lease=False,
                )

        workforce_sync.assert_not_called()
        tracker_factory.assert_not_called()
    finally:
        held_connection.close()
        engine.dispose()


@pytest.mark.asyncio
async def test_execute_task_workforce_pool_timeout_stops_tracker_checkout(
    tmp_path,
) -> None:
    """A workforce checkout timeout must not cascade into tracker checkout."""
    engine, factory, task_id = _create_single_connection_runtime_db(
        tmp_path,
        "workforce-pool-timeout.db",
    )
    held_connection = engine.connect()
    manager = AgentServiceManager()
    lease = TaskLease(
        task_id=task_id,
        runner_id="test-runner",
        run_id="test-run",
        attempt_id="test-attempt",
    )
    release_lease = MagicMock(return_value=True)
    stop_heartbeat = AsyncMock()

    try:
        with (
            patch("xagent.web.models.database._SessionLocal", factory),
            patch(
                "xagent.web.services.agent_service_manager._load_task_run_gate_user_id_isolated",
                return_value=None,
            ),
            # The share-quota gate (#973) sits between the run gate and the
            # lease acquisition; stub its checkout like the run gate's so the
            # pool timeout under test still lands on the workforce stage.
            patch(
                "xagent.web.services.agent_service_manager._load_task_public_run_quota_config_isolated",
                return_value=None,
            ),
            patch(
                "xagent.web.services.agent_service_manager.acquire_task_lease_isolated",
                return_value=lease,
            ),
            patch(
                "xagent.web.services.agent_service_manager.run_task_lease_heartbeat",
                new=AsyncMock(),
            ),
            patch(
                "xagent.web.services.agent_service_manager.stop_task_lease_heartbeat",
                new=stop_heartbeat,
            ),
            patch(
                "xagent.web.services.agent_service_manager._release_managed_task_lease_isolated",
                release_lease,
            ),
            patch(
                "xagent.web.tracking.task_tracker.TaskTracker",
            ) as tracker_factory,
            patch.object(
                manager,
                "_release_sandbox_task",
                new=AsyncMock(),
            ),
        ):
            with pytest.raises(SQLAlchemyTimeoutError, match="QueuePool limit"):
                await manager.execute_task(
                    agent_service=_FakeAgentService(),
                    task="hello",
                    tracking_task_id=str(task_id),
                    manage_task_lease=True,
                )

        tracker_factory.assert_not_called()
        stop_heartbeat.assert_awaited_once()
        release_lease.assert_not_called()
    finally:
        held_connection.close()
        engine.dispose()


@pytest.mark.asyncio
async def test_execute_task_pre_run_timeout_waits_for_shared_heartbeat_batch() -> None:
    """Cleanup shares the heartbeat waiter and never opens a release checkout."""

    task_id = 109
    lease = TaskLease(
        task_id=task_id,
        runner_id="test-runner",
        run_id="test-run",
        attempt_id="test-attempt",
    )
    manager = AgentServiceManager()
    workforce_started = threading.Event()
    heartbeat_started = threading.Event()
    allow_workforce_timeout = threading.Event()
    allow_heartbeat_timeout = threading.Event()
    workforce_timeout = SQLAlchemyTimeoutError("workforce pool exhausted")
    heartbeat_timeout = SQLAlchemyTimeoutError("heartbeat pool exhausted")
    release_lease = MagicMock(return_value=True)

    def blocking_workforce_sync(*_args, **_kwargs) -> bool:
        workforce_started.set()
        assert allow_workforce_timeout.wait(timeout=DB_PROGRESS_TIMEOUT)
        raise workforce_timeout

    def blocking_heartbeat_batch(
        _leases: tuple[TaskLease, ...],
    ) -> dict[tuple[int, str, str | None], object]:
        heartbeat_started.set()
        assert allow_heartbeat_timeout.wait(timeout=DB_PROGRESS_TIMEOUT)
        raise heartbeat_timeout

    with (
        patch(
            "xagent.web.services.agent_service_manager._load_task_run_gate_user_id_isolated",
            return_value=None,
        ),
        patch(
            "xagent.web.services.agent_service_manager.acquire_task_lease_isolated",
            return_value=lease,
        ),
        patch(
            "xagent.web.services.agent_service_manager.sync_workforce_run_status_for_task_id_isolated",
            side_effect=blocking_workforce_sync,
        ),
        patch(
            "xagent.web.services.agent_service_manager._release_managed_task_lease_isolated",
            release_lease,
        ),
        patch(
            "xagent.web.tracking.task_tracker.TaskTracker",
        ) as tracker_factory,
        patch.object(
            task_lease_service,
            "get_task_lease_heartbeat_seconds",
            return_value=0.001,
        ),
        patch.object(
            task_lease_service,
            "refresh_task_leases_isolated",
            side_effect=blocking_heartbeat_batch,
        ),
        patch.object(manager, "_release_sandbox_task", new=AsyncMock()),
    ):
        execution = asyncio.create_task(
            manager.execute_task(
                agent_service=_FakeAgentService(),
                task="hello",
                tracking_task_id=str(task_id),
                manage_task_lease=True,
            )
        )
        try:
            assert await asyncio.to_thread(workforce_started.wait, DB_PROGRESS_TIMEOUT)
            assert await asyncio.to_thread(heartbeat_started.wait, DB_PROGRESS_TIMEOUT)

            allow_workforce_timeout.set()
            await asyncio.sleep(0.02)
            assert not execution.done()
            assert not allow_heartbeat_timeout.is_set()

            allow_heartbeat_timeout.set()
            with pytest.raises(SQLAlchemyTimeoutError) as exc_info:
                await asyncio.wait_for(execution, timeout=DB_PROGRESS_TIMEOUT)
            assert exc_info.value is workforce_timeout
        finally:
            allow_workforce_timeout.set()
            allow_heartbeat_timeout.set()
            if not execution.done():
                execution.cancel()
                await asyncio.gather(execution, return_exceptions=True)
            await asyncio.wait_for(
                task_lease_service.wait_for_heartbeat_manager_idle(),
                timeout=DB_PROGRESS_TIMEOUT,
            )

    tracker_factory.assert_not_called()
    release_lease.assert_not_called()


@pytest.mark.asyncio
async def test_execute_task_waits_for_shared_heartbeat_timeout_and_retains_lease() -> (
    None
):
    """A shared heartbeat timeout must not fan out into a release checkout."""

    task_id = 110
    lease = TaskLease(
        task_id=task_id,
        runner_id="test-runner",
        run_id="test-run",
        attempt_id="test-attempt",
    )
    manager = AgentServiceManager()
    usage_started = threading.Event()
    heartbeat_started = threading.Event()
    allow_usage = threading.Event()
    allow_heartbeat_timeout = threading.Event()
    heartbeat_timeout = SQLAlchemyTimeoutError("heartbeat pool exhausted")
    release_lease = MagicMock(return_value=True)

    # The thread waits below are hang detectors: under xdist CI load the
    # execution has missed a 1 s bound to reach usage settlement. The test
    # releases every gate itself, so a long bound costs nothing when it passes.
    def blocking_usage(*_args, **_kwargs) -> bool:
        usage_started.set()
        assert allow_usage.wait(timeout=DB_PROGRESS_TIMEOUT)
        return True

    def blocking_heartbeat_batch(
        _leases: tuple[TaskLease, ...],
    ) -> dict[tuple[int, str, str | None], object]:
        heartbeat_started.set()
        assert allow_heartbeat_timeout.wait(timeout=DB_PROGRESS_TIMEOUT)
        raise heartbeat_timeout

    with (
        patch(
            "xagent.web.services.agent_service_manager._load_task_run_gate_user_id_isolated",
            return_value=None,
        ),
        patch(
            "xagent.web.services.agent_service_manager.acquire_task_lease_isolated",
            return_value=lease,
        ),
        patch(
            "xagent.web.services.agent_service_manager.sync_workforce_run_status_for_task_id_isolated",
            return_value=False,
        ),
        patch(
            "xagent.web.services.agent_service_manager._release_managed_task_lease_isolated",
            release_lease,
        ),
        patch(
            "xagent.web.tracking.task_tracker._load_task_seed_sync",
            return_value=_TaskTrackingSeed(user_id=7, usage=TokenUsage()),
        ),
        patch(
            "xagent.web.tracking.task_tracker._complete_task_usage_sync",
            side_effect=blocking_usage,
        ),
        patch.object(
            task_lease_service,
            "get_task_lease_heartbeat_seconds",
            return_value=0.001,
        ),
        patch.object(
            task_lease_service,
            "refresh_task_leases_isolated",
            side_effect=blocking_heartbeat_batch,
        ),
        patch.object(
            manager,
            "_acquire_sandbox_task",
            new=AsyncMock(return_value=None),
        ),
        patch.object(manager, "_release_sandbox_task", new=AsyncMock()),
    ):
        execution = asyncio.create_task(
            manager.execute_task(
                agent_service=_FakeAgentService(),
                task="hello",
                tracking_task_id=str(task_id),
                manage_task_lease=True,
            )
        )
        try:
            assert await asyncio.to_thread(usage_started.wait, DB_PROGRESS_TIMEOUT)
            assert await asyncio.to_thread(heartbeat_started.wait, DB_PROGRESS_TIMEOUT)

            allow_usage.set()
            await asyncio.sleep(0.02)
            assert not execution.done()
            assert not allow_heartbeat_timeout.is_set()

            allow_heartbeat_timeout.set()
            with pytest.raises(SQLAlchemyTimeoutError) as exc_info:
                await asyncio.wait_for(execution, timeout=DB_PROGRESS_TIMEOUT)
            assert exc_info.value is heartbeat_timeout
        finally:
            allow_usage.set()
            allow_heartbeat_timeout.set()
            if not execution.done():
                execution.cancel()
                await asyncio.gather(execution, return_exceptions=True)
            await asyncio.wait_for(
                task_lease_service.wait_for_heartbeat_manager_idle(),
                timeout=DB_PROGRESS_TIMEOUT,
            )

    release_lease.assert_not_called()


@pytest.mark.asyncio
async def test_execute_task_tracker_pool_timeout_stops_execution_and_release() -> None:
    task_id = 108
    lease = TaskLease(
        task_id=task_id,
        runner_id="test-runner",
        run_id="test-run",
        attempt_id="test-attempt",
    )
    manager = AgentServiceManager()
    agent_service = _FakeAgentService()
    agent_service.execute_task = AsyncMock()  # type: ignore[method-assign]
    tracker = MagicMock()
    tracker.start_tracking = AsyncMock(
        side_effect=SQLAlchemyTimeoutError("tracker pool exhausted")
    )
    release_lease = MagicMock(return_value=True)

    with (
        patch(
            "xagent.web.services.agent_service_manager._load_task_run_gate_user_id_isolated",
            return_value=None,
        ),
        patch(
            "xagent.web.services.agent_service_manager.acquire_task_lease_isolated",
            return_value=lease,
        ),
        patch(
            "xagent.web.services.agent_service_manager.sync_workforce_run_status_for_task_id_isolated",
            return_value=False,
        ),
        patch(
            "xagent.web.services.agent_service_manager.run_task_lease_heartbeat",
            new=AsyncMock(),
        ),
        patch(
            "xagent.web.services.agent_service_manager.stop_task_lease_heartbeat",
            new=AsyncMock(),
        ),
        patch(
            "xagent.web.services.agent_service_manager._release_managed_task_lease_isolated",
            release_lease,
        ),
        patch(
            "xagent.web.tracking.task_tracker.TaskTracker",
            return_value=tracker,
        ),
        patch.object(manager, "_release_sandbox_task", new=AsyncMock()),
    ):
        with pytest.raises(SQLAlchemyTimeoutError, match="tracker pool exhausted"):
            await manager.execute_task(
                agent_service=agent_service,
                task="hello",
                tracking_task_id=str(task_id),
                manage_task_lease=True,
            )

    agent_service.execute_task.assert_not_awaited()
    release_lease.assert_not_called()


@pytest.mark.asyncio
async def test_execute_task_non_pool_quota_error_remains_fail_open() -> None:
    manager = AgentServiceManager()
    agent_service = _FakeAgentService()

    with (
        patch(
            "xagent.web.services.agent_service_manager._load_task_run_gate_user_id_isolated",
            return_value=7,
        ),
        patch(
            "xagent.web.services.agent_service_manager._check_task_run_gate_on_event_loop",
            side_effect=RuntimeError("quota service unavailable"),
        ),
        patch(
            "xagent.web.services.agent_service_manager.sync_workforce_run_status_for_task_id_isolated",
            return_value=False,
        ),
        patch(
            "xagent.web.tracking.task_tracker.TaskTracker",
            side_effect=RuntimeError("tracking unavailable"),
        ),
        patch.object(
            manager,
            "_acquire_sandbox_task",
            new=AsyncMock(return_value=None),
        ),
        patch.object(manager, "_release_sandbox_task", new=AsyncMock()),
    ):
        result = await manager.execute_task(
            agent_service=agent_service,
            task="hello",
            tracking_task_id="101",
            manage_task_lease=False,
        )

    assert result["success"] is True


@pytest.mark.asyncio
async def test_execute_task_preflight_pool_wait_does_not_block_event_loop(
    tmp_path,
) -> None:
    """Run-gate context/workforce preflight waits off the event loop."""
    engine = create_engine(
        f"sqlite:///{tmp_path / 'execute-preflight-pool.db'}",
        poolclass=QueuePool,
        pool_size=1,
        max_overflow=0,
        pool_timeout=0.5,
        connect_args={"check_same_thread": False},
    )
    User.__table__.create(bind=engine)
    Task.__table__.create(bind=engine)
    factory = sessionmaker(bind=engine, autocommit=False, autoflush=False)
    with factory() as setup_db:
        user = User(username="pool-user", password_hash="hash", is_admin=False)
        setup_db.add(user)
        setup_db.flush()
        task = Task(
            user_id=user.id,
            title="pool test",
            description="test",
            status=TaskStatus.RUNNING,
            execution_mode="auto",
        )
        setup_db.add(task)
        setup_db.commit()
        task_id = int(task.id)

    held_connection = engine.connect()
    caller_db = factory()
    manager = AgentServiceManager()
    ticks = 0
    stop = asyncio.Event()

    async def ticker() -> None:
        nonlocal ticks
        while not stop.is_set():
            ticks += 1
            await asyncio.sleep(0.01)

    execute = None
    with gated_pool_checkout(engine) as gate:
        ticker_task = asyncio.create_task(ticker())
        try:
            with (
                patch("xagent.web.models.database._SessionLocal", factory),
                patch.object(
                    manager, "_acquire_sandbox_task", new=AsyncMock(return_value=None)
                ),
                patch.object(manager, "_release_sandbox_task", new=AsyncMock()),
                patch(
                    "xagent.web.tracking.task_tracker.TaskTracker",
                    side_effect=RuntimeError("skip tracking in pool-boundary test"),
                ),
            ):
                execute = asyncio.create_task(
                    manager.execute_task(
                        agent_service=_FakeAgentService(),
                        task="hello",
                        tracking_task_id=str(task_id),
                        db_session=caller_db,
                        manage_task_lease=False,
                    )
                )
                await gate.wait_until_contending()
                observed = await wait_for_ticks(lambda: ticks)
                assert observed >= LOOP_LIVENESS_TICKS
                assert not execute.done()

                held_connection.close()
                gate.let_through()
                result = await asyncio.wait_for(execute, timeout=GUARD_TIMEOUT)
                assert result["success"] is True
        finally:
            if not held_connection.closed:
                held_connection.close()
                gate.let_through()
            if execute is not None:
                await asyncio.wait_for(
                    asyncio.gather(execute, return_exceptions=True),
                    timeout=GUARD_TIMEOUT,
                )
            stop.set()
            await ticker_task
            caller_db.close()
            engine.dispose()


@pytest.mark.asyncio
async def test_execute_task_releases_read_only_caller_checkout_before_worker_io(
    tmp_path,
) -> None:
    """A legacy caller checkout must not force the worker to need slot two."""
    engine, factory, task_id = _create_single_connection_runtime_db(
        tmp_path,
        "execute-caller-checkout.db",
    )
    caller_db = factory()
    manager = AgentServiceManager()
    gate_checked_out: list[int] = []

    # A SELECT starts a transaction and pins the pool's only connection.
    caller_db.query(Task).filter(Task.id == task_id).one()
    assert engine.pool.checkedout() == 1

    def isolated_gate(worker_task_id: int) -> None:
        gate_checked_out.append(engine.pool.checkedout())
        with factory() as worker_db:
            worker_db.query(Task).filter(Task.id == worker_task_id).one()
        return None

    try:
        with (
            patch(
                "xagent.web.services.agent_service_manager._load_task_run_gate_user_id_isolated",
                side_effect=isolated_gate,
            ),
            patch(
                "xagent.web.services.agent_service_manager.sync_workforce_run_status_for_task_id_isolated",
                return_value=False,
            ),
            patch(
                "xagent.web.tracking.task_tracker.TaskTracker",
                side_effect=RuntimeError("skip tracking in pool-boundary test"),
            ),
            patch.object(
                manager,
                "_acquire_sandbox_task",
                new=AsyncMock(return_value=None),
            ),
            patch.object(manager, "_release_sandbox_task", new=AsyncMock()),
        ):
            result = await manager.execute_task(
                agent_service=_FakeAgentService(),
                task="hello",
                tracking_task_id=str(task_id),
                db_session=caller_db,
                manage_task_lease=False,
            )

        assert result["success"] is True
        assert gate_checked_out == [0]
        assert not caller_db.in_transaction()
    finally:
        caller_db.close()
        engine.dispose()


@pytest.mark.asyncio
async def test_execute_task_rejects_dirty_caller_session_before_worker_io(
    tmp_path,
) -> None:
    """Pending caller writes must never be rolled back to free a pool slot."""
    engine, factory, task_id = _create_single_connection_runtime_db(
        tmp_path,
        "execute-dirty-caller.db",
    )
    caller_db = factory()
    manager = AgentServiceManager()
    task = caller_db.query(Task).filter(Task.id == task_id).one()
    task.title = "uncommitted caller title"

    try:
        with (
            patch(
                "xagent.web.services.agent_service_manager._load_task_run_gate_user_id_isolated",
            ) as quota_gate,
            pytest.raises(
                RuntimeError,
                match="caller database session has pending writes",
            ),
        ):
            await manager.execute_task(
                agent_service=_FakeAgentService(),
                task="hello",
                tracking_task_id=str(task_id),
                db_session=caller_db,
                manage_task_lease=False,
            )

        quota_gate.assert_not_called()
        assert task in caller_db.dirty
        assert task.title == "uncommitted caller title"
    finally:
        caller_db.rollback()
        caller_db.close()
        engine.dispose()


@pytest.mark.asyncio
# Shared execution accepts a run whose lease execute_task acquires itself.
@pytest.mark.parametrize("shared", [False, True])
async def test_execute_task_acquires_and_releases_lease_when_manage_true(
    db_session, shared: bool
) -> None:
    user = User(username="lease-user", password_hash="hash", is_admin=False)
    db_session.add(user)
    db_session.commit()
    task = Task(
        user_id=user.id,
        title="lease test",
        description="test",
        status=TaskStatus.RUNNING,
        execution_mode="auto",
    )
    db_session.add(task)
    db_session.commit()

    fake_lease = TaskLease(
        task_id=int(task.id),
        runner_id="test-runner",
        run_id="test-run",
        attempt_id="test-attempt",
    )
    manager = AgentServiceManager()

    with (
        patch(
            "xagent.web.services.agent_service_manager."
            "get_shared_task_execution_enabled",
            return_value=shared,
        ),
        patch(
            "xagent.web.services.agent_service_manager.acquire_task_lease_isolated",
            return_value=fake_lease,
        ) as mock_acquire,
        patch(
            "xagent.web.services.agent_service_manager.release_task_lease_with_workforce_sync",
        ) as mock_release,
        patch(
            "xagent.web.services.agent_service_manager.run_task_lease_heartbeat",
            new=AsyncMock(),
        ),
        patch(
            "xagent.web.services.agent_service_manager.stop_task_lease_heartbeat",
            new=AsyncMock(),
        ),
        patch.object(
            manager, "_acquire_sandbox_task", new=AsyncMock(return_value=None)
        ),
        patch.object(manager, "_release_sandbox_task", new=AsyncMock()),
        patch(
            "xagent.web.services.agent_service_manager.sync_workforce_run_status_for_task_id_isolated",
            return_value=False,
        ) as mock_sync,
    ):
        result = await manager.execute_task(
            agent_service=_FakeAgentService(),
            task="hello",
            tracking_task_id=str(task.id),
            db_session=db_session,
            manage_task_lease=True,
        )

    assert result["success"] is True
    mock_acquire.assert_called_once()
    mock_release.assert_called_once()
    mock_sync.assert_called_once_with(
        int(task.id),
        TaskStatus.RUNNING,
        task_lease=fake_lease,
    )


@pytest.mark.asyncio
async def test_execute_task_skips_lease_but_syncs_running_when_manage_false(
    db_session,
) -> None:
    user = User(username="lease-user2", password_hash="hash", is_admin=False)
    db_session.add(user)
    db_session.commit()
    task = Task(
        user_id=user.id,
        title="lease test",
        description="test",
        status=TaskStatus.RUNNING,
        execution_mode="auto",
    )
    db_session.add(task)
    db_session.commit()

    manager = AgentServiceManager()

    with (
        patch(
            "xagent.web.services.agent_service_manager.acquire_task_lease_isolated",
        ) as mock_acquire,
        patch(
            "xagent.web.services.agent_service_manager.release_task_lease_with_workforce_sync",
        ) as mock_release,
        patch(
            "xagent.web.services.agent_service_manager.run_task_lease_heartbeat",
            new=AsyncMock(),
        ),
        patch(
            "xagent.web.services.agent_service_manager.stop_task_lease_heartbeat",
            new=AsyncMock(),
        ) as mock_stop_hb,
        patch.object(
            manager, "_acquire_sandbox_task", new=AsyncMock(return_value=None)
        ),
        patch.object(manager, "_release_sandbox_task", new=AsyncMock()),
        patch(
            "xagent.web.services.agent_service_manager.sync_workforce_run_status_for_task_id_isolated",
            return_value=False,
        ) as mock_sync,
        patch(
            "xagent.web.tracking.task_tracker.TaskTracker",
            side_effect=RuntimeError("skip tracking in unit test"),
        ),
    ):
        result = await manager.execute_task(
            agent_service=_FakeAgentService(),
            task="hello",
            tracking_task_id=str(task.id),
            db_session=db_session,
            manage_task_lease=False,
        )

    assert result["success"] is True
    mock_acquire.assert_not_called()
    mock_release.assert_not_called()
    mock_sync.assert_called_once_with(
        int(task.id),
        TaskStatus.RUNNING,
        task_lease=None,
    )
    mock_stop_hb.assert_awaited_once_with(None, None)


@pytest.mark.asyncio
async def test_execute_task_surfaces_mid_run_quota_reason(db_session) -> None:
    """When the mid-run quota gate trips, the run result is reshaped to a
    terminal quota_exceeded carrying the reason as output (mirroring the start
    gate) instead of the pattern-interrupt path's silent flip to PAUSED."""
    user = User(username="quota-user", password_hash="hash", is_admin=False)
    db_session.add(user)
    db_session.commit()
    task = Task(
        user_id=user.id,
        title="quota test",
        description="test",
        status=TaskStatus.RUNNING,
        execution_mode="auto",
    )
    db_session.add(task)
    db_session.commit()

    manager = AgentServiceManager()
    reason = "Monthly ai_credits_per_month quota reached. Upgrade your plan."
    tracker = MagicMock()
    tracker.start_tracking = AsyncMock()
    tracker.complete_tracking = AsyncMock()
    tracker.quota_interrupt_reason = reason  # the mid-run gate tripped

    agent_service = _FakeAgentService()
    # The pattern-interrupt path returns a silent "interrupted" result.
    agent_service.execute_task = AsyncMock(  # type: ignore[method-assign]
        return_value={"success": False, "status": "interrupted", "error": "interrupted"}
    )

    with (
        patch(
            "xagent.web.services.agent_service_manager.run_task_lease_heartbeat",
            new=AsyncMock(),
        ),
        patch(
            "xagent.web.services.agent_service_manager.stop_task_lease_heartbeat",
            new=AsyncMock(),
        ),
        patch.object(
            manager, "_acquire_sandbox_task", new=AsyncMock(return_value=None)
        ),
        patch.object(manager, "_release_sandbox_task", new=AsyncMock()),
        patch(
            "xagent.web.services.agent_service_manager.sync_workforce_run_status_for_task_id_isolated",
            return_value=False,
        ),
        patch(
            "xagent.web.tracking.task_tracker.TaskTracker",
            return_value=tracker,
        ),
    ):
        result = await manager.execute_task(
            agent_service=agent_service,
            task="hello",
            tracking_task_id=str(task.id),
            db_session=db_session,
            manage_task_lease=False,
        )

    assert result["status"] == "quota_exceeded"
    assert result["success"] is False
    assert result["output"] == reason
    assert result["error"] == reason
    # A mid-run interrupt is always the quota checker, so the result carries the
    # code (matching the start gate) to drive the app-layer dialog.
    assert result["error_code"] == "quota_exceeded"


@pytest.mark.asyncio
async def test_execute_task_start_gate_forwards_structured_reason(db_session) -> None:
    """When the start gate returns a structured reason (mapping), the run result
    carries its message plus error_code/error_details so the client can localise
    and branch, instead of only a plain string."""
    user = User(username="quota-start-user", password_hash="hash", is_admin=False)
    db_session.add(user)
    db_session.commit()
    task = Task(
        user_id=user.id,
        title="quota start test",
        description="test",
        status=TaskStatus.PENDING,
        execution_mode="auto",
    )
    db_session.add(task)
    db_session.commit()

    manager = AgentServiceManager()
    event_loop_thread = threading.get_ident()
    hook_threads: list[int] = []
    block = {
        "code": "quota_exceeded",
        "metric": "runs_per_month",
        "limit": 0,
        "plan": "basic",
        "message": "Team quota exhausted for this billing period.",
    }

    def run_gate(*_args):
        hook_threads.append(threading.get_ident())
        return block

    # The gate short-circuits before lease/tracker/execution, so a patched
    # check_run_gate returning the structured block is enough.
    with patch(
        "xagent.web.services.quota_hooks.check_run_gate",
        side_effect=run_gate,
    ):
        result = await manager.execute_task(
            agent_service=_FakeAgentService(),
            task="hello",
            tracking_task_id=str(task.id),
            db_session=db_session,
            manage_task_lease=False,
        )

    assert result["status"] == "quota_exceeded"
    assert result["success"] is False
    assert result["output"] == block["message"]
    assert result["error_code"] == "quota_exceeded"
    assert result["error_details"] == block
    assert hook_threads == [event_loop_thread]


@pytest.mark.asyncio
async def test_execute_task_cancellation_during_workforce_sync_releases_lease() -> None:
    """Cancellation after heartbeat start must drain worker I/O then clean up."""
    task_id = 106
    lease = TaskLease(
        task_id=task_id,
        runner_id="test-runner",
        run_id="test-run",
        attempt_id="test-attempt",
    )
    manager = AgentServiceManager()
    workforce_sync_started = threading.Event()
    allow_workforce_sync = threading.Event()
    release_lease = MagicMock(return_value=True)
    stop_heartbeat = AsyncMock()
    release_sandbox = AsyncMock()

    def blocking_workforce_sync(*_args, **_kwargs) -> bool:
        workforce_sync_started.set()
        if not allow_workforce_sync.wait(timeout=1):
            raise AssertionError("workforce sync test worker was not released")
        return False

    with (
        patch(
            "xagent.web.services.agent_service_manager._load_task_run_gate_user_id_isolated",
            return_value=None,
        ),
        patch(
            "xagent.web.services.agent_service_manager.acquire_task_lease_isolated",
            return_value=lease,
        ),
        patch(
            "xagent.web.services.agent_service_manager.sync_workforce_run_status_for_task_id_isolated",
            side_effect=blocking_workforce_sync,
        ),
        patch(
            "xagent.web.services.agent_service_manager._release_managed_task_lease_isolated",
            release_lease,
        ),
        patch(
            "xagent.web.services.agent_service_manager.run_task_lease_heartbeat",
            new=AsyncMock(),
        ),
        patch(
            "xagent.web.services.agent_service_manager.stop_task_lease_heartbeat",
            new=stop_heartbeat,
        ),
        patch(
            "xagent.web.tracking.task_tracker.TaskTracker",
        ) as tracker_factory,
        patch.object(
            manager,
            "_acquire_sandbox_task",
            new=AsyncMock(return_value=None),
        ) as acquire_sandbox,
        patch.object(
            manager,
            "_release_sandbox_task",
            new=release_sandbox,
        ),
    ):
        execution = asyncio.create_task(
            manager.execute_task(
                agent_service=_FakeAgentService(),
                task="hello",
                tracking_task_id=str(task_id),
                manage_task_lease=True,
            )
        )
        started = await asyncio.wait_for(
            asyncio.to_thread(workforce_sync_started.wait, 1),
            timeout=1,
        )
        assert started

        execution.cancel()
        await asyncio.sleep(0)
        assert not execution.done()
        allow_workforce_sync.set()

        with pytest.raises(asyncio.CancelledError):
            await asyncio.wait_for(execution, timeout=1)

    tracker_factory.assert_not_called()
    acquire_sandbox.assert_not_awaited()
    stop_heartbeat.assert_awaited_once()
    release_lease.assert_called_once()
    assert release_lease.call_args.kwargs["status"] == TaskStatus.FAILED
    release_sandbox.assert_awaited_once_with(None)


@pytest.mark.asyncio
async def test_execute_task_cancellation_during_tracker_start_releases_lease() -> None:
    """Tracker startup is inside the lease owner's cleanup boundary."""
    task_id = 107
    lease = TaskLease(
        task_id=task_id,
        runner_id="test-runner",
        run_id="test-run",
        attempt_id="test-attempt",
    )
    manager = AgentServiceManager()
    tracker_start_entered = asyncio.Event()
    tracker = MagicMock()
    tracker.quota_interrupt_reason = None

    async def blocking_tracker_start() -> None:
        tracker_start_entered.set()
        await asyncio.Event().wait()

    tracker.start_tracking = AsyncMock(side_effect=blocking_tracker_start)
    tracker.complete_tracking = AsyncMock()
    release_lease = MagicMock(return_value=True)
    stop_heartbeat = AsyncMock()
    release_sandbox = AsyncMock()

    with (
        patch(
            "xagent.web.services.agent_service_manager._load_task_run_gate_user_id_isolated",
            return_value=None,
        ),
        patch(
            "xagent.web.services.agent_service_manager.acquire_task_lease_isolated",
            return_value=lease,
        ),
        patch(
            "xagent.web.services.agent_service_manager.sync_workforce_run_status_for_task_id_isolated",
            return_value=False,
        ),
        patch(
            "xagent.web.services.agent_service_manager._release_managed_task_lease_isolated",
            release_lease,
        ),
        patch(
            "xagent.web.services.agent_service_manager.run_task_lease_heartbeat",
            new=AsyncMock(),
        ),
        patch(
            "xagent.web.services.agent_service_manager.stop_task_lease_heartbeat",
            new=stop_heartbeat,
        ),
        patch(
            "xagent.web.tracking.task_tracker.TaskTracker",
            return_value=tracker,
        ),
        patch.object(
            manager,
            "_acquire_sandbox_task",
            new=AsyncMock(return_value=None),
        ) as acquire_sandbox,
        patch.object(
            manager,
            "_release_sandbox_task",
            new=release_sandbox,
        ),
    ):
        execution = asyncio.create_task(
            manager.execute_task(
                agent_service=_FakeAgentService(),
                task="hello",
                tracking_task_id=str(task_id),
                manage_task_lease=True,
            )
        )
        await asyncio.wait_for(tracker_start_entered.wait(), timeout=1)
        execution.cancel()

        with pytest.raises(asyncio.CancelledError):
            await asyncio.wait_for(execution, timeout=1)

    acquire_sandbox.assert_not_awaited()
    tracker.complete_tracking.assert_awaited_once()
    stop_heartbeat.assert_awaited_once()
    release_lease.assert_called_once()
    assert release_lease.call_args.kwargs["status"] == TaskStatus.FAILED
    release_sandbox.assert_awaited_once_with(None)


@pytest.mark.asyncio
async def test_execute_task_cleans_up_when_sandbox_acquire_raises(
    db_session,
) -> None:
    """A reclaimed-sandbox raise from ``_acquire_sandbox_task`` must still
    run the finally cleanup: heartbeat stop, lease release, and tracker
    completion (whose only call site is that finally block)."""
    user = User(username="lease-user3", password_hash="hash", is_admin=False)
    db_session.add(user)
    db_session.commit()
    task = Task(
        user_id=user.id,
        title="lease test",
        description="test",
        status=TaskStatus.RUNNING,
        execution_mode="auto",
    )
    db_session.add(task)
    db_session.commit()

    fake_lease = TaskLease(
        task_id=int(task.id),
        runner_id="test-runner",
        run_id="test-run",
        attempt_id="test-attempt",
    )
    manager = AgentServiceManager()
    tracker = MagicMock()
    tracker.start_tracking = AsyncMock()
    tracker.complete_tracking = AsyncMock()
    agent_service = _FakeAgentService()
    agent_service.execute_task = AsyncMock()  # type: ignore[method-assign]
    agent_service.set_interrupt_checker = MagicMock()  # type: ignore[method-assign]

    with (
        patch(
            "xagent.web.services.agent_service_manager.acquire_task_lease_isolated",
            return_value=fake_lease,
        ),
        patch(
            "xagent.web.services.agent_service_manager.release_task_lease_with_workforce_sync",
        ) as mock_release,
        patch(
            "xagent.web.services.agent_service_manager.run_task_lease_heartbeat",
            new=AsyncMock(),
        ),
        patch(
            "xagent.web.services.agent_service_manager.stop_task_lease_heartbeat",
            new=AsyncMock(),
        ) as mock_stop_hb,
        patch.object(
            manager,
            "_acquire_sandbox_task",
            new=AsyncMock(side_effect=RuntimeError("sandbox reclaimed")),
        ),
        patch.object(manager, "_release_sandbox_task", new=AsyncMock()) as mock_sbx,
        patch(
            "xagent.web.services.agent_service_manager.sync_workforce_run_status_for_task_id_isolated",
            return_value=False,
        ),
        patch(
            "xagent.web.tracking.task_tracker.TaskTracker",
            return_value=tracker,
        ),
    ):
        with pytest.raises(RuntimeError, match="sandbox reclaimed"):
            await manager.execute_task(
                agent_service=agent_service,
                task="hello",
                tracking_task_id=str(task.id),
                db_session=db_session,
                manage_task_lease=True,
            )

    agent_service.execute_task.assert_not_awaited()
    mock_stop_hb.assert_awaited_once()
    mock_release.assert_called_once()
    assert mock_release.call_args.kwargs["status"] == TaskStatus.FAILED
    tracker.complete_tracking.assert_awaited_once()
    mock_sbx.assert_awaited_once_with(None)
    # The mid-run quota checker must be cleared in the finally so a reused
    # agent_service can't keep calling this finished run's tracker.
    agent_service.set_interrupt_checker.assert_any_call(None)


@pytest.mark.asyncio
async def test_execute_task_persists_final_usage_before_releasing_lease() -> None:
    """The final usage snapshot belongs to the current run, so it must land
    while that run still owns its lease."""
    task_id = 101
    lease = TaskLease(
        task_id=task_id,
        runner_id="test-runner",
        run_id="test-run",
        attempt_id="test-attempt",
    )
    manager = AgentServiceManager()
    events: list[str] = []

    tracker = MagicMock()
    tracker.start_tracking = AsyncMock()
    tracker.quota_interrupt_reason = None

    async def complete_tracking() -> None:
        events.append("usage")

    tracker.complete_tracking = AsyncMock(side_effect=complete_tracking)

    async def stop_heartbeat(*_args) -> None:
        events.append("heartbeat")

    def release_lease(*_args, **_kwargs) -> bool:
        events.append("lease")
        return True

    async def release_sandbox(*_args) -> None:
        events.append("sandbox")

    with (
        patch(
            "xagent.web.services.agent_service_manager._load_task_run_gate_user_id_isolated",
            return_value=None,
        ),
        patch(
            "xagent.web.services.agent_service_manager.acquire_task_lease_isolated",
            return_value=lease,
        ),
        patch(
            "xagent.web.services.agent_service_manager.sync_workforce_run_status_for_task_id_isolated",
            return_value=False,
        ),
        patch(
            "xagent.web.services.agent_service_manager._release_managed_task_lease_isolated",
            side_effect=release_lease,
        ),
        patch(
            "xagent.web.services.agent_service_manager.run_task_lease_heartbeat",
            new=AsyncMock(),
        ),
        patch(
            "xagent.web.services.agent_service_manager.stop_task_lease_heartbeat",
            new=stop_heartbeat,
        ),
        patch(
            "xagent.web.tracking.task_tracker.TaskTracker",
            return_value=tracker,
        ) as tracker_factory,
        patch.object(
            manager,
            "_acquire_sandbox_task",
            new=AsyncMock(return_value="user:101"),
        ),
        patch.object(manager, "_release_sandbox_task", new=release_sandbox),
    ):
        result = await manager.execute_task(
            agent_service=_FakeAgentService(),
            task="hello",
            tracking_task_id=str(task_id),
            manage_task_lease=True,
        )

    assert result["success"] is True
    tracker_factory.assert_called_once_with(
        task_id=task_id,
        expected_run_id="test-run",
        expected_runner_id="test-runner",
        expected_attempt_id="test-attempt",
    )
    assert events == ["usage", "heartbeat", "lease", "sandbox"]


@pytest.mark.asyncio
async def test_execute_task_releases_sandbox_when_lease_release_raises() -> None:
    """A lease settlement failure must not strand the independent sandbox."""
    task_id = 105
    lease = TaskLease(
        task_id=task_id,
        runner_id="test-runner",
        run_id="test-run",
        attempt_id="test-attempt",
    )
    manager = AgentServiceManager()
    tracker = MagicMock()
    tracker.start_tracking = AsyncMock()
    tracker.complete_tracking = AsyncMock()
    tracker.quota_interrupt_reason = None
    release_sandbox = AsyncMock()

    with (
        patch(
            "xagent.web.services.agent_service_manager._load_task_run_gate_user_id_isolated",
            return_value=None,
        ),
        patch(
            "xagent.web.services.agent_service_manager.acquire_task_lease_isolated",
            return_value=lease,
        ),
        patch(
            "xagent.web.services.agent_service_manager.sync_workforce_run_status_for_task_id_isolated",
            return_value=False,
        ),
        patch(
            "xagent.web.services.agent_service_manager._release_managed_task_lease_isolated",
            side_effect=RuntimeError("lease release failed"),
        ),
        patch(
            "xagent.web.services.agent_service_manager.run_task_lease_heartbeat",
            new=AsyncMock(),
        ),
        patch(
            "xagent.web.services.agent_service_manager.stop_task_lease_heartbeat",
            new=AsyncMock(),
        ),
        patch(
            "xagent.web.tracking.task_tracker.TaskTracker",
            return_value=tracker,
        ),
        patch.object(
            manager,
            "_acquire_sandbox_task",
            new=AsyncMock(return_value="user:105"),
        ),
        patch.object(manager, "_release_sandbox_task", new=release_sandbox),
    ):
        with pytest.raises(RuntimeError, match="lease release failed"):
            await manager.execute_task(
                agent_service=_FakeAgentService(),
                task="hello",
                tracking_task_id=str(task_id),
                manage_task_lease=True,
            )

    release_sandbox.assert_awaited_once_with("user:105")


@pytest.mark.asyncio
async def test_execute_task_final_usage_pool_timeout_retains_lease() -> None:
    """One exhausted final checkout must not trigger a second lease checkout."""
    task_id = 103
    lease = TaskLease(
        task_id=task_id,
        runner_id="test-runner",
        run_id="test-run",
        attempt_id="test-attempt",
    )
    manager = AgentServiceManager()
    tracker = MagicMock()
    tracker.start_tracking = AsyncMock()
    tracker.complete_tracking = AsyncMock(
        side_effect=SQLAlchemyTimeoutError("pool exhausted")
    )
    tracker.quota_interrupt_reason = None

    release_lease = MagicMock()
    stop_heartbeat = AsyncMock()
    release_sandbox = AsyncMock()

    with (
        patch(
            "xagent.web.services.agent_service_manager._load_task_run_gate_user_id_isolated",
            return_value=None,
        ),
        patch(
            "xagent.web.services.agent_service_manager.acquire_task_lease_isolated",
            return_value=lease,
        ),
        patch(
            "xagent.web.services.agent_service_manager.sync_workforce_run_status_for_task_id_isolated",
            return_value=False,
        ),
        patch(
            "xagent.web.services.agent_service_manager._release_managed_task_lease_isolated",
            release_lease,
        ),
        patch(
            "xagent.web.services.agent_service_manager.run_task_lease_heartbeat",
            new=AsyncMock(),
        ),
        patch(
            "xagent.web.services.agent_service_manager.stop_task_lease_heartbeat",
            new=stop_heartbeat,
        ),
        patch(
            "xagent.web.tracking.task_tracker.TaskTracker",
            return_value=tracker,
        ),
        patch.object(
            manager,
            "_acquire_sandbox_task",
            new=AsyncMock(return_value="user:103"),
        ),
        patch.object(manager, "_release_sandbox_task", new=release_sandbox),
    ):
        with pytest.raises(SQLAlchemyTimeoutError, match="pool exhausted"):
            await manager.execute_task(
                agent_service=_FakeAgentService(),
                task="hello",
                tracking_task_id=str(task_id),
                manage_task_lease=True,
            )

    tracker.complete_tracking.assert_awaited_once()
    stop_heartbeat.assert_awaited_once()
    release_lease.assert_not_called()
    release_sandbox.assert_awaited_once_with("user:103")


@pytest.mark.asyncio
async def test_execute_task_heartbeat_pool_timeout_retains_lease() -> None:
    """Heartbeat pool exhaustion must not be followed by lease release I/O."""
    task_id = 104
    lease = TaskLease(
        task_id=task_id,
        runner_id="test-runner",
        run_id="test-run",
        attempt_id="test-attempt",
    )
    manager = AgentServiceManager()
    tracker = MagicMock()
    tracker.start_tracking = AsyncMock()
    tracker.complete_tracking = AsyncMock()
    tracker.quota_interrupt_reason = None
    heartbeat_timeout = SQLAlchemyTimeoutError("heartbeat pool exhausted")
    release_lease = MagicMock()
    release_sandbox = AsyncMock()

    with (
        patch(
            "xagent.web.services.agent_service_manager._load_task_run_gate_user_id_isolated",
            return_value=None,
        ),
        patch(
            "xagent.web.services.agent_service_manager.acquire_task_lease_isolated",
            return_value=lease,
        ),
        patch(
            "xagent.web.services.agent_service_manager.sync_workforce_run_status_for_task_id_isolated",
            return_value=False,
        ),
        patch(
            "xagent.web.services.agent_service_manager._release_managed_task_lease_isolated",
            release_lease,
        ),
        patch(
            "xagent.web.services.agent_service_manager.run_task_lease_heartbeat",
            new=AsyncMock(),
        ),
        patch(
            "xagent.web.services.agent_service_manager.stop_task_lease_heartbeat",
            new=AsyncMock(
                return_value=TaskLeaseHeartbeatOutcome(pool_timeout=heartbeat_timeout)
            ),
        ),
        patch(
            "xagent.web.tracking.task_tracker.TaskTracker",
            return_value=tracker,
        ),
        patch.object(
            manager,
            "_acquire_sandbox_task",
            new=AsyncMock(return_value="user:104"),
        ),
        patch.object(manager, "_release_sandbox_task", new=release_sandbox),
    ):
        with pytest.raises(SQLAlchemyTimeoutError, match="heartbeat pool exhausted"):
            await manager.execute_task(
                agent_service=_FakeAgentService(),
                task="hello",
                tracking_task_id=str(task_id),
                manage_task_lease=True,
            )

    tracker.complete_tracking.assert_awaited_once()
    release_lease.assert_not_called()
    release_sandbox.assert_awaited_once_with("user:104")


@pytest.mark.asyncio
async def test_execute_task_heartbeat_loss_after_result_rejects_success() -> None:
    task_id = 106
    lease = TaskLease(
        task_id=task_id,
        runner_id="test-runner",
        run_id="test-run",
        attempt_id="test-attempt",
    )
    manager = AgentServiceManager()
    tracker = MagicMock()
    tracker.start_tracking = AsyncMock()
    tracker.complete_tracking = AsyncMock()
    tracker.stop_periodic_updates = AsyncMock()
    tracker.quota_interrupt_reason = None
    release_lease = MagicMock()

    with (
        patch(
            "xagent.web.services.agent_service_manager._load_task_run_gate_user_id_isolated",
            return_value=None,
        ),
        patch(
            "xagent.web.services.agent_service_manager.acquire_task_lease_isolated",
            return_value=lease,
        ),
        patch(
            "xagent.web.services.agent_service_manager.sync_workforce_run_status_for_task_id_isolated",
            return_value=False,
        ),
        patch(
            "xagent.web.services.agent_service_manager._release_managed_task_lease_isolated",
            release_lease,
        ),
        patch(
            "xagent.web.services.agent_service_manager.run_task_lease_heartbeat",
            new=AsyncMock(),
        ),
        patch(
            "xagent.web.services.agent_service_manager.stop_task_lease_heartbeat",
            new=AsyncMock(return_value=TaskLeaseHeartbeatOutcome(lease_lost=True)),
        ),
        patch(
            "xagent.web.tracking.task_tracker.TaskTracker",
            return_value=tracker,
        ),
        patch.object(
            manager,
            "_acquire_sandbox_task",
            new=AsyncMock(return_value="user:106"),
        ),
        patch.object(manager, "_release_sandbox_task", new=AsyncMock()),
    ):
        with pytest.raises(TaskLeaseLostError):
            await manager.execute_task(
                agent_service=_FakeAgentService(),
                task="hello",
                tracking_task_id=str(task_id),
                manage_task_lease=True,
            )

    release_lease.assert_not_called()


def test_task_title_update_is_fenced_by_exact_lease(db_session) -> None:
    user = User(username="title-fence-user", password_hash="hash", is_admin=False)
    db_session.add(user)
    db_session.flush()
    task = Task(
        user_id=user.id,
        title="Original title",
        status=TaskStatus.RUNNING,
        runner_id="current-runner",
        run_id="current-run",
        lease_attempt_id="test-attempt",
    )
    db_session.add(task)
    db_session.commit()

    stale = TaskLease(
        task_id=int(task.id),
        runner_id="stale-runner",
        run_id="stale-run",
        attempt_id="test-attempt",
    )
    current = TaskLease(
        task_id=int(task.id),
        runner_id="current-runner",
        run_id="current-run",
        attempt_id="test-attempt",
    )

    assert (
        _update_task_title_isolated(
            int(task.id),
            "Stale title",
            task_lease=stale,
        )
        is False
    )
    db_session.expire_all()
    assert db_session.get(Task, task.id).title == "Original title"

    assert (
        _update_task_title_isolated(
            int(task.id),
            "Current title",
            task_lease=current,
        )
        is True
    )
    db_session.expire_all()
    assert db_session.get(Task, task.id).title == "Current title"


@pytest.mark.asyncio
async def test_execute_task_cancellation_during_heartbeat_stop_drains_cleanup() -> None:
    """Caller cancellation during heartbeat shutdown must not strand the
    lease or skip tracker and sandbox cleanup."""
    task_id = 102
    lease = TaskLease(
        task_id=task_id,
        runner_id="test-runner",
        run_id="test-run",
        attempt_id="test-attempt",
    )
    manager = AgentServiceManager()
    events: list[str] = []
    heartbeat_stop_entered = asyncio.Event()
    allow_heartbeat_stop = asyncio.Event()

    tracker = MagicMock()
    tracker.start_tracking = AsyncMock()
    tracker.quota_interrupt_reason = None

    async def complete_tracking() -> None:
        events.append("usage")

    tracker.complete_tracking = AsyncMock(side_effect=complete_tracking)

    async def stop_heartbeat(*_args) -> None:
        events.append("heartbeat-enter")
        heartbeat_stop_entered.set()
        await allow_heartbeat_stop.wait()
        events.append("heartbeat-finish")

    def release_lease(*_args, **_kwargs) -> bool:
        events.append("lease")
        return True

    async def release_sandbox(*_args) -> None:
        events.append("sandbox")

    with (
        patch(
            "xagent.web.services.agent_service_manager._load_task_run_gate_user_id_isolated",
            return_value=None,
        ),
        patch(
            "xagent.web.services.agent_service_manager.acquire_task_lease_isolated",
            return_value=lease,
        ),
        patch(
            "xagent.web.services.agent_service_manager.sync_workforce_run_status_for_task_id_isolated",
            return_value=False,
        ),
        patch(
            "xagent.web.services.agent_service_manager._release_managed_task_lease_isolated",
            side_effect=release_lease,
        ),
        patch(
            "xagent.web.services.agent_service_manager.run_task_lease_heartbeat",
            new=AsyncMock(),
        ),
        patch(
            "xagent.web.services.agent_service_manager.stop_task_lease_heartbeat",
            new=stop_heartbeat,
        ),
        patch(
            "xagent.web.tracking.task_tracker.TaskTracker",
            return_value=tracker,
        ),
        patch.object(
            manager,
            "_acquire_sandbox_task",
            new=AsyncMock(return_value="user:102"),
        ),
        patch.object(manager, "_release_sandbox_task", new=release_sandbox),
    ):
        execution = asyncio.create_task(
            manager.execute_task(
                agent_service=_FakeAgentService(),
                task="hello",
                tracking_task_id=str(task_id),
                manage_task_lease=True,
            )
        )
        await asyncio.wait_for(heartbeat_stop_entered.wait(), timeout=1)
        execution.cancel()
        await asyncio.sleep(0)
        allow_heartbeat_stop.set()

        with pytest.raises(asyncio.CancelledError):
            await asyncio.wait_for(execution, timeout=1)

    assert events == [
        "usage",
        "heartbeat-enter",
        "heartbeat-finish",
        "lease",
        "sandbox",
    ]


def test_sync_workforce_run_status_running_is_idempotent(db_session) -> None:
    """Repeat RUNNING sync is a no-op when WorkforceRun is already running."""
    user = User(username="sync-user", password_hash="hash", is_admin=False)
    db_session.add(user)
    db_session.flush()
    manager = Agent(
        user_id=user.id,
        name="Manager",
        description="desc",
        instructions="instr",
        execution_mode="balanced",
        models={"general": "test-model"},
        knowledge_bases=[],
        skills=[],
        tool_categories=[],
        suggested_prompts=[],
        status=AgentStatus.PUBLISHED,
    )
    db_session.add(manager)
    db_session.flush()
    workforce = Workforce(
        owner_user_id=user.id,
        scope_type="user",
        scope_id=str(user.id),
        name="Team",
        description="desc",
        manager_agent_id=manager.id,
        status="active",
    )
    db_session.add(workforce)
    db_session.flush()
    task = Task(
        user_id=user.id,
        title="sync test",
        description="test",
        status=TaskStatus.RUNNING,
        agent_id=manager.id,
        agent_config={},
        execution_mode="auto",
    )
    db_session.add(task)
    db_session.flush()
    run = WorkforceRun(
        workforce_id=workforce.id,
        task_id=task.id,
        user_id=user.id,
        status="running",
        snapshot={"version": 1},
    )
    db_session.add(run)
    db_session.flush()
    task.agent_config = {"workforce_run_id": run.id}
    db_session.commit()

    assert sync_workforce_run_status(db_session, task, TaskStatus.RUNNING) is False
    assert sync_workforce_run_status(db_session, task, TaskStatus.RUNNING) is False
    db_session.refresh(run)
    assert run.status == "running"
    assert run.completed_at is None


@pytest.mark.parametrize("use_stale_lease", [False, True])
def test_delayed_workforce_running_projection_cannot_resurrect_terminal_run(
    db_session,
    use_stale_lease: bool,
) -> None:
    user = User(username="projection-user", password_hash="hash", is_admin=False)
    db_session.add(user)
    db_session.flush()
    manager = Agent(
        user_id=user.id,
        name="Projection manager",
        description="desc",
        instructions="instr",
        execution_mode="balanced",
        models={"general": "test-model"},
        knowledge_bases=[],
        skills=[],
        tool_categories=[],
        suggested_prompts=[],
        status=AgentStatus.PUBLISHED,
    )
    db_session.add(manager)
    db_session.flush()
    workforce = Workforce(
        owner_user_id=user.id,
        scope_type="user",
        scope_id=str(user.id),
        name="Projection team",
        description="desc",
        manager_agent_id=manager.id,
        status="active",
    )
    db_session.add(workforce)
    db_session.flush()
    task = Task(
        user_id=user.id,
        title="terminal projection",
        description="test",
        status=TaskStatus.COMPLETED,
        runner_id="replacement-b",
        run_id="same-run",
        agent_id=manager.id,
        agent_config={},
        execution_mode="auto",
        lease_attempt_id="test-attempt",
    )
    db_session.add(task)
    db_session.flush()
    run = WorkforceRun(
        workforce_id=workforce.id,
        task_id=task.id,
        user_id=user.id,
        status="completed",
        completed_at=datetime.now(timezone.utc),
        snapshot={"version": 1},
    )
    db_session.add(run)
    db_session.flush()
    task.agent_config = {"workforce_run_id": int(run.id)}
    db_session.commit()

    stale_lease = (
        TaskLease(
            task_id=int(task.id),
            runner_id="stale-a",
            run_id="same-run",
            attempt_id="test-attempt",
        )
        if use_stale_lease
        else None
    )
    assert (
        sync_workforce_run_status_for_task_id_isolated(
            int(task.id),
            TaskStatus.RUNNING,
            task_lease=stale_lease,
        )
        is False
    )

    db_session.expire_all()
    persisted_run = db_session.get(WorkforceRun, int(run.id))
    assert persisted_run.status == "completed"
    assert persisted_run.completed_at is not None


# --- unsettled evidence for a pooled release ------------------------------------
#
# A lease this manager acquired itself (``manage_task_lease=True``) is either
# released here or left to TTL recovery. The scenarios above already drive
# each outcome with the real ``execute_task``; re-run unchanged, the process
# hold's sticky ``unsettled_total`` must grow by one exactly when the lease
# was not released, and never on a confirmed release.

UNSETTLED_SCENARIOS = [
    (test_execute_task_persists_final_usage_before_releasing_lease, 0),
    (test_execute_task_cancellation_during_workforce_sync_releases_lease, 0),
    (test_execute_task_cancellation_during_tracker_start_releases_lease, 0),
    (test_execute_task_cleans_up_when_sandbox_acquire_raises, 0),
    (test_execute_task_managed_lease_loss_skips_usage_and_release, 1),
    (test_execute_task_workforce_pool_timeout_stops_tracker_checkout, 1),
    (test_execute_task_pre_run_timeout_waits_for_shared_heartbeat_batch, 1),
    (test_execute_task_waits_for_shared_heartbeat_timeout_and_retains_lease, 1),
    (test_execute_task_tracker_pool_timeout_stops_execution_and_release, 1),
    (test_execute_task_releases_sandbox_when_lease_release_raises, 1),
    (test_execute_task_final_usage_pool_timeout_retains_lease, 1),
    (test_execute_task_heartbeat_pool_timeout_retains_lease, 1),
    (test_execute_task_heartbeat_loss_after_result_rejects_success, 1),
]


@pytest.mark.asyncio
@pytest.mark.parametrize(
    ("scenario", "expected"),
    UNSETTLED_SCENARIOS,
    ids=[scenario.__name__ for scenario, _ in UNSETTLED_SCENARIOS],
)
async def test_managed_execute_records_unsettled_only_when_the_lease_is_retained(
    request: pytest.FixtureRequest, scenario, expected: int
) -> None:
    import inspect

    from xagent.web.services import release_hold as release_hold_module

    hold = release_hold_module.release_hold
    kwargs = {
        name: request.getfixturevalue(name)
        for name in inspect.signature(scenario).parameters
    }
    before = hold.unsettled_total
    await scenario(**kwargs)
    assert hold.unsettled_total - before == expected
    if expected:
        assert hold.snapshot()["unsettled"][-1]["kind"] == "managed_execution_lease"