Skip to content

Enterprise agent platform for building, deploying, and operating AI teammates.

55.7 KB
Raw Blame
"""Tests for task execution leases."""

import asyncio
import logging
import re
import threading
import uuid
from datetime import timedelta

import pytest
from sqlalchemy import create_engine, event
from sqlalchemy.exc import TimeoutError as SQLAlchemyTimeoutError
from sqlalchemy.orm import sessionmaker
from sqlalchemy.pool import QueuePool

from tests.shared.db_teardown import drop_all_tables
from tests.web.pool_contention_shared import (
    GUARD_TIMEOUT,
    LOOP_LIVENESS_TICKS,
    gated_pool_checkout,
    wait_for_ticks,
)
from xagent.core.agent.checkpoint import CHECKPOINT_TYPE, LEGACY_CHECKPOINT_TYPES
from xagent.web.models import database as database_module
from xagent.web.models.database import Base, get_db, get_engine, init_db
from xagent.web.models.task import ExecutionMode, Task, TaskStatus, TraceEvent
from xagent.web.models.user import User
from xagent.web.services import task_lease_service
from xagent.web.services.db_runtime import drain_async_task_cancellation_safe
from xagent.web.services.task_lease_recovery import (
    recover_task_lease_candidate_isolated,
)
from xagent.web.services.task_lease_service import (
    TASK_RUN_ID_TRACE_FIELD,
    TaskLease,
    TaskLeaseHeartbeatOutcome,
    TaskLeaseLostError,
    TaskLeaseRefreshState,
    acquire_task_lease,
    acquire_task_lease_no_commit,
    get_expired_task_lease_candidates,
    refresh_task_lease,
    release_task_lease,
    run_task_lease_heartbeat,
    run_while_task_lease_owned,
    stop_task_lease_heartbeat,
    utc_now,
)


@pytest.fixture()
def db_session(tmp_path):
    init_db(db_url=f"sqlite:///{tmp_path / 'lease.db'}")
    db = next(get_db())
    try:
        yield db
    finally:
        db.close()
        drop_all_tables(get_engine())


@pytest.fixture()
def queue_pool_runtime_db(tmp_path):
    """A real one-slot QueuePool used to exercise checkout contention."""
    engine = create_engine(
        f"sqlite:///{tmp_path / 'lease-queue-pool.db'}",
        connect_args={"check_same_thread": False},
        poolclass=QueuePool,
        pool_size=1,
        max_overflow=0,
        pool_timeout=0.4,
    )
    Base.metadata.create_all(bind=engine)
    SessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine)
    try:
        yield engine, SessionLocal
    finally:
        engine.dispose()


def _create_task(db, *, status=TaskStatus.PENDING) -> Task:
    user = User(username="lease-user", password_hash="hash", is_admin=False)
    db.add(user)
    db.commit()
    db.refresh(user)
    task = Task(
        user_id=user.id,
        title="Lease test",
        description="Lease test",
        status=status,
        execution_mode="auto",
    )
    db.add(task)
    db.commit()
    db.refresh(task)
    return task


def _recover_expired_task(db, task: Task) -> TaskStatus | None:
    candidate = get_expired_task_lease_candidates(
        db,
        cutoff=utc_now(),
        limit=1,
    )[0]
    db.rollback()
    return recover_task_lease_candidate_isolated(
        candidate,
        recovered_at=utc_now(),
    )


@pytest.mark.asyncio
async def test_guard_cancels_and_drains_execution_on_lease_loss() -> None:
    operation_started = asyncio.Event()
    operation_cancelled = asyncio.Event()

    async def operation() -> None:
        operation_started.set()
        try:
            await asyncio.Event().wait()
        finally:
            operation_cancelled.set()

    async def heartbeat() -> TaskLeaseHeartbeatOutcome:
        await operation_started.wait()
        return TaskLeaseHeartbeatOutcome(lease_lost=True)

    heartbeat_task = asyncio.create_task(heartbeat())
    with pytest.raises(TaskLeaseLostError):
        await run_while_task_lease_owned(operation(), heartbeat_task)

    assert operation_cancelled.is_set()
    assert heartbeat_task.done()


@pytest.mark.asyncio
async def test_guard_keeps_execution_running_for_settlement_ready() -> None:
    operation_started = asyncio.Event()
    allow_operation_to_finish = asyncio.Event()

    async def operation() -> str:
        operation_started.set()
        await allow_operation_to_finish.wait()
        return "completed"

    async def heartbeat() -> TaskLeaseHeartbeatOutcome:
        await operation_started.wait()
        allow_operation_to_finish.set()
        return TaskLeaseHeartbeatOutcome()

    heartbeat_task = asyncio.create_task(heartbeat())

    assert await run_while_task_lease_owned(operation(), heartbeat_task) == "completed"


@pytest.mark.asyncio
async def test_guard_does_not_cancel_for_transient_heartbeat_pool_timeout() -> None:
    timeout_observed = asyncio.Event()
    stop_heartbeat = asyncio.Event()

    async def heartbeat() -> TaskLeaseHeartbeatOutcome:
        timeout_observed.set()
        await stop_heartbeat.wait()
        return TaskLeaseHeartbeatOutcome(
            pool_timeout=SQLAlchemyTimeoutError("transient pool timeout")
        )

    async def operation() -> str:
        await timeout_observed.wait()
        return "kept-running"

    heartbeat_task = asyncio.create_task(heartbeat())
    assert (
        await run_while_task_lease_owned(operation(), heartbeat_task) == "kept-running"
    )
    assert not heartbeat_task.done()
    stop_heartbeat.set()
    outcome = await heartbeat_task
    assert outcome.pool_timeout is not None


@pytest.mark.asyncio
async def test_guard_cancels_and_drains_execution_when_heartbeat_crashes() -> None:
    operation_started = asyncio.Event()
    operation_cancelled = asyncio.Event()

    async def operation() -> None:
        operation_started.set()
        try:
            await asyncio.Event().wait()
        finally:
            operation_cancelled.set()

    async def heartbeat() -> TaskLeaseHeartbeatOutcome:
        await operation_started.wait()
        raise RuntimeError("heartbeat crashed")

    heartbeat_task = asyncio.create_task(heartbeat())
    with pytest.raises(RuntimeError, match="heartbeat crashed"):
        await run_while_task_lease_owned(operation(), heartbeat_task)

    assert operation_cancelled.is_set()
    assert heartbeat_task.done()


def test_task_model_default_execution_mode_is_auto(db_session) -> None:
    user = User(username="default-mode-user", password_hash="hash", is_admin=False)
    db_session.add(user)
    db_session.commit()
    db_session.refresh(user)

    task = Task(
        user_id=user.id,
        title="Default mode",
        description="Default mode",
        status=TaskStatus.PENDING,
    )
    db_session.add(task)
    db_session.commit()
    db_session.refresh(task)

    assert task.execution_mode == "auto"
    assert task.execution_mode_enum == ExecutionMode.AUTO


def test_task_lease_acquire_refresh_and_release(db_session) -> None:
    task = _create_task(db_session)

    lease = acquire_task_lease(db_session, int(task.id), runner_id="runner-a")

    assert lease is not None
    db_session.refresh(task)
    assert task.status == TaskStatus.RUNNING
    assert task.runner_id == "runner-a"
    assert task.lease_expires_at is not None
    assert task.run_id == lease.run_id
    assert task.state_version == 1
    assert task.control_state == "running"

    assert acquire_task_lease(db_session, int(task.id), runner_id="runner-b") is None
    assert refresh_task_lease(db_session, lease) == TaskLeaseRefreshState.REFRESHED
    assert release_task_lease(db_session, lease, status=TaskStatus.COMPLETED) is True
    db_session.refresh(task)
    assert task.status == TaskStatus.COMPLETED
    assert task.state_version == 2
    assert task.control_state == "completed"
    assert task.runner_id is None
    assert task.lease_expires_at is None


def test_refresh_owned_terminal_task_is_settlement_ready(db_session) -> None:
    task = _create_task(db_session)
    lease = acquire_task_lease(db_session, int(task.id), runner_id="runner-a")
    assert lease is not None

    task.status = TaskStatus.COMPLETED
    task.control_state = "completed"
    db_session.commit()

    assert (
        refresh_task_lease(db_session, lease) == TaskLeaseRefreshState.SETTLEMENT_READY
    )


def test_acquire_returns_run_id_from_update_without_followup_select(
    queue_pool_runtime_db,
) -> None:
    engine, SessionLocal = queue_pool_runtime_db
    with SessionLocal() as seed_db:
        task = _create_task(seed_db)
        task_id = int(task.id)

    statements: list[str] = []

    def record_statement(
        _conn,
        _cursor,
        statement: str,
        _parameters,
        _context,
        _executemany,
    ) -> None:
        statements.append(statement)

    event.listen(engine, "before_cursor_execute", record_statement)
    try:
        with SessionLocal() as db:
            lease = acquire_task_lease(db, task_id, runner_id="runner-returning")
    finally:
        event.remove(engine, "before_cursor_execute", record_statement)

    assert lease is not None
    assert lease.run_id
    assert len(statements) == 1
    assert statements[0].lstrip().upper().startswith("UPDATE")
    assert "RETURNING" in statements[0].upper()


def test_refresh_batch_uses_one_pool_checkout(
    queue_pool_runtime_db,
    monkeypatch,
) -> None:
    engine, SessionLocal = queue_pool_runtime_db
    leases: list[TaskLease] = []
    with SessionLocal() as seed_db:
        user = User(
            username="batch-heartbeat-owner",
            password_hash="hash",
            is_admin=False,
        )
        seed_db.add(user)
        seed_db.flush()
        for index in range(100):
            run_id = f"run-{index}"
            task = Task(
                user_id=user.id,
                title=f"Batch lease {index}",
                description="Batch lease",
                status=TaskStatus.RUNNING,
                execution_mode="auto",
                runner_id="runner-a",
                run_id=run_id,
                lease_attempt_id="test-attempt",
            )
            seed_db.add(task)
            seed_db.flush()
            leases.append(
                TaskLease(
                    task_id=int(task.id),
                    runner_id="runner-a",
                    run_id=run_id,
                    attempt_id="test-attempt",
                )
            )
        seed_db.commit()

    monkeypatch.setattr(database_module, "_SessionLocal", SessionLocal)
    checkouts = 0

    def record_checkout(*_args) -> None:
        nonlocal checkouts
        checkouts += 1

    event.listen(engine, "checkout", record_checkout)
    try:
        states = task_lease_service.refresh_task_leases_isolated(tuple(leases))
    finally:
        event.remove(engine, "checkout", record_checkout)

    assert checkouts == 1
    assert len(states) == 100
    assert set(states.values()) == {TaskLeaseRefreshState.REFRESHED}


def test_acquire_no_commit_leaves_transaction_owned_by_caller(db_session) -> None:
    task = _create_task(db_session)

    lease = acquire_task_lease_no_commit(
        db_session,
        int(task.id),
        runner_id="transaction-owner",
    )

    assert lease is not None
    db_session.rollback()
    db_session.refresh(task)
    assert task.status == TaskStatus.PENDING
    assert task.runner_id is None
    assert task.run_id is None


def test_fail_and_release_task_lease_rejects_superseded_owner(db_session) -> None:
    task = _create_task(db_session)
    stale_lease = acquire_task_lease(
        db_session,
        int(task.id),
        runner_id="old-runner",
    )
    assert stale_lease is not None

    task.runner_id = "new-runner"
    task.lease_attempt_id = "test-attempt"
    task.run_id = "new-run"
    task.error_message = None
    task.output = "new owner output"
    db_session.commit()

    changed = task_lease_service.fail_and_release_task_lease_no_commit(
        db_session,
        stale_lease,
        error_message="stale runner failed",
    )
    db_session.commit()

    assert changed is False
    db_session.refresh(task)
    assert task.status == TaskStatus.RUNNING
    assert task.runner_id == "new-runner"
    assert task.run_id == "new-run"
    assert task.error_message is None
    assert task.output == "new owner output"


def test_fail_and_release_task_lease_atomically_fails_current_owner(db_session) -> None:
    task = _create_task(db_session)
    lease = acquire_task_lease(db_session, int(task.id), runner_id="runner-a")
    assert lease is not None
    task.output = "stale output"
    db_session.commit()
    state_version = int(task.state_version)

    changed = task_lease_service.fail_and_release_task_lease_no_commit(
        db_session,
        lease,
        error_message="setup failed",
    )
    db_session.commit()

    assert changed is True
    db_session.refresh(task)
    assert task.status == TaskStatus.FAILED
    assert task.control_state == "failed"
    assert task.state_version == state_version + 1
    assert task.error_message == "setup failed"
    assert task.output is None
    assert task.runner_id is None
    assert task.lease_expires_at is None


def test_release_task_lease_refuses_ownerless_running_state(db_session) -> None:
    task = _create_task(db_session)
    lease = acquire_task_lease(db_session, int(task.id), runner_id="runner-a")
    assert lease is not None

    with pytest.raises(ValueError, match="RUNNING"):
        release_task_lease(db_session, lease, status=TaskStatus.RUNNING)

    db_session.refresh(task)
    assert task.status == TaskStatus.RUNNING
    assert task.runner_id == "runner-a"
    assert task.lease_expires_at is not None


@pytest.mark.parametrize(
    "status",
    [TaskStatus.PAUSED, TaskStatus.WAITING_FOR_USER],
)
def test_release_to_non_terminal_status_clears_stale_error_message(
    db_session, status
) -> None:
    task = _create_task(db_session)
    lease = acquire_task_lease(db_session, int(task.id), runner_id="runner-a")
    assert lease is not None
    task.error_message = "earlier failure"
    task.output = "prior answer"
    db_session.commit()

    changed = task_lease_service.release_task_lease_no_commit(
        db_session,
        lease,
        status=status,
    )
    db_session.commit()

    assert changed is True
    db_session.refresh(task)
    assert task.status == status
    assert task.error_message is None
    assert task.output == "prior answer"


@pytest.mark.parametrize(
    "status",
    [TaskStatus.COMPLETED, TaskStatus.FAILED],
)
def test_release_to_terminal_status_leaves_content_fields_alone(
    db_session, status
) -> None:
    task = _create_task(db_session)
    lease = acquire_task_lease(db_session, int(task.id), runner_id="runner-a")
    assert lease is not None
    task.error_message = "earlier failure"
    task.output = "prior answer"
    db_session.commit()

    changed = task_lease_service.release_task_lease_no_commit(
        db_session,
        lease,
        status=status,
    )
    db_session.commit()

    assert changed is True
    db_session.refresh(task)
    assert task.status == status
    assert task.error_message == "earlier failure"
    assert task.output == "prior answer"


@pytest.mark.asyncio
async def test_lease_heartbeat_keeps_loop_responsive_during_pool_checkout(
    queue_pool_runtime_db,
    monkeypatch,
) -> None:
    engine, SessionLocal = queue_pool_runtime_db
    with SessionLocal() as seed_db:
        task = _create_task(seed_db, status=TaskStatus.RUNNING)
        task_id = int(task.id)
        task.runner_id = "runner-a"
        task.lease_attempt_id = "test-attempt"
        task.run_id = "run-a"
        seed_db.commit()

    def constrained_get_db():
        db = SessionLocal()
        try:
            yield db
        finally:
            db.close()

    monkeypatch.setattr(
        task_lease_service,
        "get_db",
        constrained_get_db,
        raising=False,
    )
    monkeypatch.setattr(database_module, "_SessionLocal", SessionLocal)
    monkeypatch.setattr(
        task_lease_service,
        "get_task_lease_heartbeat_seconds",
        lambda: 0.001,
    )

    held_connection = engine.connect()
    stop_event = asyncio.Event()
    ticker_stop = asyncio.Event()
    ticks = 0

    async def ticker() -> None:
        nonlocal ticks
        while not ticker_stop.is_set():
            await asyncio.sleep(0.01)
            ticks += 1

    with gated_pool_checkout(engine) as gate:
        heartbeat_task = asyncio.create_task(
            run_task_lease_heartbeat(
                TaskLease(
                    task_id=task_id,
                    runner_id="runner-a",
                    run_id="run-a",
                    attempt_id="test-attempt",
                ),
                stop_event,
            )
        )
        ticker_task = asyncio.create_task(ticker())
        try:
            await gate.wait_until_contending()
            observed = await wait_for_ticks(lambda: ticks)
            assert observed >= LOOP_LIVENESS_TICKS
        finally:
            held_connection.close()
            gate.let_through()
            stop_event.set()
            ticker_stop.set()
            await asyncio.wait_for(
                asyncio.gather(heartbeat_task, ticker_task, return_exceptions=True),
                timeout=GUARD_TIMEOUT,
            )
        heartbeat_task.result()

        with SessionLocal() as verify_db:
            refreshed = verify_db.query(Task).filter(Task.id == task_id).one()
            assert refreshed.last_heartbeat_at is not None


@pytest.mark.asyncio
async def test_stop_heartbeat_waits_for_shared_batch_result(monkeypatch) -> None:
    refresh_started = threading.Event()
    allow_refresh_to_finish = threading.Event()

    def blocking_refresh(
        leases: tuple[TaskLease, ...],
    ) -> dict[tuple[int, str, str | None], TaskLeaseRefreshState]:
        refresh_started.set()
        assert allow_refresh_to_finish.wait(timeout=GUARD_TIMEOUT)
        return {
            (lease.task_id, lease.runner_id, lease.run_id, lease.attempt_id): (
                TaskLeaseRefreshState.REFRESHED
            )
            for lease in leases
        }

    monkeypatch.setattr(
        task_lease_service,
        "refresh_task_leases_isolated",
        blocking_refresh,
    )
    monkeypatch.setattr(
        task_lease_service,
        "get_task_lease_heartbeat_seconds",
        lambda: 0.001,
    )

    stop_event = asyncio.Event()
    heartbeat_task = asyncio.create_task(
        run_task_lease_heartbeat(
            TaskLease(
                task_id=1,
                runner_id="runner-a",
                run_id="run-a",
                attempt_id="test-attempt",
            ),
            stop_event,
        )
    )
    assert await asyncio.to_thread(refresh_started.wait, GUARD_TIMEOUT)

    stopping = asyncio.create_task(
        stop_task_lease_heartbeat(heartbeat_task, stop_event)
    )
    await asyncio.sleep(0.02)
    assert not stopping.done()

    allow_refresh_to_finish.set()
    outcome = await asyncio.wait_for(stopping, timeout=GUARD_TIMEOUT)
    await task_lease_service.wait_for_heartbeat_manager_idle()

    assert outcome.requires_ttl_recovery is False


@pytest.mark.asyncio
async def test_cancelled_heartbeat_manager_settles_active_registration(
    monkeypatch,
) -> None:
    refresh_started = threading.Event()
    allow_refresh_to_finish = threading.Event()

    def blocking_refresh(
        leases: tuple[TaskLease, ...],
    ) -> dict[tuple[int, str, str | None], TaskLeaseRefreshState]:
        refresh_started.set()
        assert allow_refresh_to_finish.wait(timeout=GUARD_TIMEOUT)
        return {
            (lease.task_id, lease.runner_id, lease.run_id, lease.attempt_id): (
                TaskLeaseRefreshState.REFRESHED
            )
            for lease in leases
        }

    monkeypatch.setattr(
        task_lease_service,
        "refresh_task_leases_isolated",
        blocking_refresh,
    )
    monkeypatch.setattr(
        task_lease_service,
        "get_task_lease_heartbeat_seconds",
        lambda: 0.001,
    )

    manager = task_lease_service._TaskLeaseHeartbeatManager(asyncio.get_running_loop())
    registration = manager.register(
        TaskLease(
            task_id=1, runner_id="runner-a", run_id="run-a", attempt_id="test-attempt"
        )
    )
    assert await asyncio.to_thread(refresh_started.wait, GUARD_TIMEOUT)

    runner = manager._runner
    assert runner is not None
    runner.cancel()
    allow_refresh_to_finish.set()
    with pytest.raises(asyncio.CancelledError):
        await runner

    outcome = await asyncio.wait_for(registration.close(), timeout=GUARD_TIMEOUT)

    assert outcome.requires_ttl_recovery is False
    assert registration._entry.refresh_waiter is None


@pytest.mark.asyncio
async def test_wait_for_heartbeat_manager_idle_drains_before_caller_cancellation(
    monkeypatch,
) -> None:
    manager = task_lease_service._TaskLeaseHeartbeatManager(asyncio.get_running_loop())
    runner_started = asyncio.Event()
    allow_runner_to_finish = asyncio.Event()

    async def runner() -> None:
        runner_started.set()
        await allow_runner_to_finish.wait()

    manager._runner = asyncio.create_task(runner())
    monkeypatch.setattr(
        task_lease_service,
        "_task_lease_heartbeat_manager",
        manager,
    )
    await runner_started.wait()

    waiter = asyncio.create_task(task_lease_service.wait_for_heartbeat_manager_idle())
    await asyncio.sleep(0)
    assert not waiter.done()
    waiter.cancel()
    await asyncio.sleep(0)

    assert not waiter.done()
    assert manager._runner is not None
    assert not manager._runner.done()

    allow_runner_to_finish.set()
    with pytest.raises(asyncio.CancelledError):
        await waiter


@pytest.mark.asyncio
async def test_repeated_cancellation_drains_heartbeat_close_and_waiters(
    monkeypatch,
) -> None:
    refresh_started = threading.Event()
    allow_refresh_to_finish = threading.Event()
    gather_started = asyncio.Event()
    allow_gather_to_finish = asyncio.Event()
    original_gather = asyncio.gather

    def blocking_refresh(
        leases: tuple[TaskLease, ...],
    ) -> dict[tuple[int, str, str | None], TaskLeaseRefreshState]:
        refresh_started.set()
        assert allow_refresh_to_finish.wait(timeout=GUARD_TIMEOUT)
        return {
            (lease.task_id, lease.runner_id, lease.run_id, lease.attempt_id): (
                TaskLeaseRefreshState.REFRESHED
            )
            for lease in leases
        }

    async def blocking_gather(*args, **kwargs):
        gather_started.set()
        await allow_gather_to_finish.wait()
        return await original_gather(*args, **kwargs)

    monkeypatch.setattr(
        task_lease_service,
        "refresh_task_leases_isolated",
        blocking_refresh,
    )
    monkeypatch.setattr(
        task_lease_service,
        "get_task_lease_heartbeat_seconds",
        lambda: 0.001,
    )
    monkeypatch.setattr(task_lease_service.asyncio, "gather", blocking_gather)

    heartbeat_task = asyncio.get_running_loop().create_task(
        run_task_lease_heartbeat(
            TaskLease(
                task_id=1,
                runner_id="runner-a",
                run_id="run-a",
                attempt_id="test-attempt",
            ),
            asyncio.Event(),
        )
    )
    assert await asyncio.to_thread(refresh_started.wait, GUARD_TIMEOUT)

    manager = task_lease_service._get_task_lease_heartbeat_manager()
    entry = next(iter(manager._entries.values()))
    refresh_waiter = entry.refresh_waiter
    assert refresh_waiter is not None

    heartbeat_task.cancel()
    await asyncio.sleep(0.02)
    assert not heartbeat_task.done()

    allow_refresh_to_finish.set()
    await asyncio.wait_for(gather_started.wait(), timeout=GUARD_TIMEOUT)

    heartbeat_task.cancel()
    await asyncio.sleep(0.02)
    try:
        assert not heartbeat_task.done()
    finally:
        allow_gather_to_finish.set()

    with pytest.raises(asyncio.CancelledError):
        await asyncio.wait_for(heartbeat_task, timeout=GUARD_TIMEOUT)
    await task_lease_service.wait_for_heartbeat_manager_idle()

    assert refresh_waiter.done()
    assert entry.refresh_waiter is None
    assert manager._entries == {}


@pytest.mark.asyncio
async def test_heartbeat_requires_exact_run_id() -> None:
    lease = TaskLease(task_id=1, runner_id="runner-a", attempt_id="test-attempt")

    with pytest.raises(ValueError, match="exact run_id fence"):
        await run_task_lease_heartbeat(lease, asyncio.Event())


@pytest.fixture
def heartbeat_batch_ready(monkeypatch):
    """Start the shared runner only after both test leases are registered."""
    manager_type = task_lease_service._TaskLeaseHeartbeatManager
    register = manager_type.register
    run = manager_type._run
    ready = asyncio.Event()
    registrations = 0

    def register_and_signal(manager, lease):
        nonlocal registrations
        registration = register(manager, lease)
        registrations += 1
        if registrations == 2:
            ready.set()
        return registration

    async def run_when_ready(manager):
        await asyncio.wait_for(ready.wait(), timeout=GUARD_TIMEOUT)
        await run(manager)

    monkeypatch.setattr(manager_type, "register", register_and_signal)
    monkeypatch.setattr(manager_type, "_run", run_when_ready)


@pytest.mark.asyncio
async def test_heartbeat_batches_registered_leases(
    monkeypatch, heartbeat_batch_ready
) -> None:
    refreshed = threading.Event()
    batches: list[tuple[TaskLease, ...]] = []

    def refresh_batch(
        leases: tuple[TaskLease, ...],
    ) -> dict[tuple[int, str, str | None], TaskLeaseRefreshState]:
        batches.append(leases)
        refreshed.set()
        return {
            (lease.task_id, lease.runner_id, lease.run_id, lease.attempt_id): (
                TaskLeaseRefreshState.REFRESHED
            )
            for lease in leases
        }

    monkeypatch.setattr(
        task_lease_service,
        "refresh_task_leases_isolated",
        refresh_batch,
    )
    monkeypatch.setattr(
        task_lease_service,
        "get_task_lease_heartbeat_seconds",
        lambda: 0.001,
    )

    first_stop = asyncio.Event()
    second_stop = asyncio.Event()
    first_task = asyncio.create_task(
        run_task_lease_heartbeat(
            TaskLease(
                task_id=1,
                runner_id="runner-a",
                run_id="run-a",
                attempt_id="test-attempt",
            ),
            first_stop,
        )
    )
    second_task = asyncio.create_task(
        run_task_lease_heartbeat(
            TaskLease(
                task_id=2,
                runner_id="runner-a",
                run_id="run-b",
                attempt_id="test-attempt",
            ),
            second_stop,
        )
    )
    try:
        assert await asyncio.to_thread(refreshed.wait, GUARD_TIMEOUT)
    finally:
        first_stop.set()
        second_stop.set()
        await asyncio.gather(
            stop_task_lease_heartbeat(first_task, first_stop),
            stop_task_lease_heartbeat(second_task, second_stop),
        )
        await task_lease_service.wait_for_heartbeat_manager_idle()

    # Both registered leases must share the first worker checkout.
    assert batches
    assert {(lease.task_id, lease.run_id) for lease in batches[0]} == {
        (1, "run-a"),
        (2, "run-b"),
    }


@pytest.mark.asyncio
async def test_old_batch_result_does_not_contaminate_replacement_registration(
    monkeypatch,
) -> None:
    first_refresh_started = threading.Event()
    allow_first_refresh = threading.Event()
    second_refresh_started = threading.Event()
    allow_second_refresh = threading.Event()
    attempts = 0
    lease = TaskLease(
        task_id=1, runner_id="runner-a", run_id="run-a", attempt_id="test-attempt"
    )
    key = (lease.task_id, lease.runner_id, lease.run_id, lease.attempt_id)

    def refresh_batch(
        _leases: tuple[TaskLease, ...],
    ) -> dict[tuple[int, str, str | None], TaskLeaseRefreshState]:
        nonlocal attempts
        attempts += 1
        if attempts == 1:
            first_refresh_started.set()
            assert allow_first_refresh.wait(timeout=GUARD_TIMEOUT)
            return {key: TaskLeaseRefreshState.LOST}
        second_refresh_started.set()
        assert allow_second_refresh.wait(timeout=GUARD_TIMEOUT)
        return {key: TaskLeaseRefreshState.REFRESHED}

    monkeypatch.setattr(
        task_lease_service,
        "refresh_task_leases_isolated",
        refresh_batch,
    )
    monkeypatch.setattr(
        task_lease_service,
        "get_task_lease_heartbeat_seconds",
        lambda: 0.001,
    )

    first_stop = asyncio.Event()
    first_task = asyncio.create_task(run_task_lease_heartbeat(lease, first_stop))
    assert await asyncio.to_thread(first_refresh_started.wait, GUARD_TIMEOUT)

    first_stopping = asyncio.create_task(
        stop_task_lease_heartbeat(first_task, first_stop)
    )
    await asyncio.sleep(0.02)
    assert not first_stopping.done()

    replacement_stop = asyncio.Event()
    replacement_task = asyncio.create_task(
        run_task_lease_heartbeat(lease, replacement_stop)
    )
    allow_first_refresh.set()

    first_outcome = await asyncio.wait_for(first_stopping, timeout=GUARD_TIMEOUT)
    assert first_outcome.lease_lost is True
    assert not replacement_task.done()
    assert await asyncio.to_thread(second_refresh_started.wait, GUARD_TIMEOUT)

    replacement_stopping = asyncio.create_task(
        stop_task_lease_heartbeat(replacement_task, replacement_stop)
    )
    await asyncio.sleep(0.02)
    assert not replacement_stopping.done()

    allow_second_refresh.set()
    replacement_outcome = await asyncio.wait_for(
        replacement_stopping,
        timeout=GUARD_TIMEOUT,
    )
    await task_lease_service.wait_for_heartbeat_manager_idle()

    assert replacement_outcome.requires_ttl_recovery is False


@pytest.mark.asyncio
async def test_stop_heartbeat_reports_shared_batch_pool_timeout(
    monkeypatch, caplog, heartbeat_batch_ready
) -> None:
    refresh_attempted = threading.Event()
    allow_timeout = threading.Event()
    heartbeat_timeout = SQLAlchemyTimeoutError("pool checkout timed out")

    def timed_out_refresh(
        _leases: tuple[TaskLease, ...],
    ) -> dict[tuple[int, str, str | None], TaskLeaseRefreshState]:
        refresh_attempted.set()
        assert allow_timeout.wait(timeout=GUARD_TIMEOUT)
        raise heartbeat_timeout

    monkeypatch.setattr(
        task_lease_service,
        "refresh_task_leases_isolated",
        timed_out_refresh,
    )
    monkeypatch.setattr(
        task_lease_service,
        "get_task_lease_heartbeat_seconds",
        lambda: 0.001,
    )

    first_stop_event = asyncio.Event()
    second_stop_event = asyncio.Event()
    first_heartbeat_task = None
    second_heartbeat_task = None
    first_stopping = None
    second_stopping = None

    async def stop_and_drain_heartbeat(
        stopping,
        heartbeat_task,
        stop_event,
    ) -> None:
        stop_event.set()
        if heartbeat_task is None:
            return
        if stopping is None:
            stopping = asyncio.create_task(
                stop_task_lease_heartbeat(heartbeat_task, stop_event)
            )
        try:
            await drain_async_task_cancellation_safe(stopping)
        finally:
            await drain_async_task_cancellation_safe(heartbeat_task)

    try:
        with caplog.at_level(logging.WARNING):
            first_heartbeat_task = asyncio.create_task(
                run_task_lease_heartbeat(
                    TaskLease(
                        task_id=1,
                        runner_id="runner-a",
                        run_id="run-a",
                        attempt_id="test-attempt",
                    ),
                    first_stop_event,
                )
            )
            second_heartbeat_task = asyncio.create_task(
                run_task_lease_heartbeat(
                    TaskLease(
                        task_id=2,
                        runner_id="runner-b",
                        run_id="run-b",
                        attempt_id="test-attempt",
                    ),
                    second_stop_event,
                )
            )
            assert await asyncio.wait_for(
                asyncio.to_thread(refresh_attempted.wait, GUARD_TIMEOUT),
                timeout=GUARD_TIMEOUT,
            )

            first_stopping = asyncio.create_task(
                stop_task_lease_heartbeat(first_heartbeat_task, first_stop_event)
            )
            second_stopping = asyncio.create_task(
                stop_task_lease_heartbeat(second_heartbeat_task, second_stop_event)
            )
            await asyncio.sleep(0.02)
            assert not first_stopping.done()
            assert not second_stopping.done()

            allow_timeout.set()
            first_outcome, second_outcome = await asyncio.wait_for(
                asyncio.gather(first_stopping, second_stopping),
                timeout=GUARD_TIMEOUT,
            )

        heartbeat_warnings = [
            record
            for record in caplog.records
            if record.levelno == logging.WARNING
            and "component=lease-heartbeat" in record.getMessage()
        ]
        assert len(heartbeat_warnings) == 1
        assert heartbeat_warnings[0].getMessage() == (
            "component=lease-heartbeat task_ids=[1,2] active_lease_count=2 "
            "database pool checkout timed out: pool checkout timed out"
        )
        for outcome in (first_outcome, second_outcome):
            assert outcome.pool_timeout is heartbeat_timeout
            assert outcome.lease_lost is False
            assert outcome.requires_ttl_recovery is True
    finally:
        allow_timeout.set()
        try:
            await stop_and_drain_heartbeat(
                first_stopping,
                first_heartbeat_task,
                first_stop_event,
            )
        finally:
            try:
                await stop_and_drain_heartbeat(
                    second_stopping,
                    second_heartbeat_task,
                    second_stop_event,
                )
            finally:
                await task_lease_service.wait_for_heartbeat_manager_idle()


@pytest.mark.asyncio
async def test_stop_heartbeat_reports_lost_ownership(monkeypatch) -> None:
    def lost_refresh(
        leases: tuple[TaskLease, ...],
    ) -> dict[tuple[int, str, str | None], TaskLeaseRefreshState]:
        return {
            (
                lease.task_id,
                lease.runner_id,
                lease.run_id,
                lease.attempt_id,
            ): TaskLeaseRefreshState.LOST
            for lease in leases
        }

    monkeypatch.setattr(
        task_lease_service,
        "refresh_task_leases_isolated",
        lost_refresh,
    )
    monkeypatch.setattr(
        task_lease_service,
        "get_task_lease_heartbeat_seconds",
        lambda: 0.001,
    )

    stop_event = asyncio.Event()
    heartbeat_task = asyncio.create_task(
        run_task_lease_heartbeat(
            TaskLease(
                task_id=1,
                runner_id="runner-a",
                run_id="run-a",
                attempt_id="test-attempt",
            ),
            stop_event,
        )
    )
    await asyncio.wait_for(heartbeat_task, timeout=GUARD_TIMEOUT)

    outcome = await stop_task_lease_heartbeat(heartbeat_task, stop_event)
    await task_lease_service.wait_for_heartbeat_manager_idle()

    assert outcome.lease_lost is True
    assert outcome.pool_timeout is None


@pytest.mark.asyncio
async def test_heartbeat_does_not_report_settlement_ready_as_lease_lost(
    monkeypatch,
) -> None:
    def settlement_ready_refresh(
        leases: tuple[TaskLease, ...],
    ) -> dict[tuple[int, str, str | None], TaskLeaseRefreshState]:
        return {
            (lease.task_id, lease.runner_id, lease.run_id, lease.attempt_id): (
                TaskLeaseRefreshState.SETTLEMENT_READY
            )
            for lease in leases
        }

    async def run_db_io_inline(operation):
        return operation()

    monkeypatch.setattr(
        task_lease_service,
        "refresh_task_leases_isolated",
        settlement_ready_refresh,
    )
    monkeypatch.setattr(
        task_lease_service,
        "run_db_io_cancellation_safe",
        run_db_io_inline,
    )
    monkeypatch.setattr(
        task_lease_service,
        "get_task_lease_heartbeat_seconds",
        lambda: 0.001,
    )

    outcome = await asyncio.wait_for(
        run_task_lease_heartbeat(
            TaskLease(
                task_id=1,
                runner_id="runner-a",
                run_id="run-a",
                attempt_id="test-attempt",
            ),
            asyncio.Event(),
        ),
        timeout=GUARD_TIMEOUT,
    )
    await task_lease_service.wait_for_heartbeat_manager_idle()

    assert outcome.lease_lost is False
    assert outcome.requires_ttl_recovery is False


@pytest.mark.asyncio
async def test_batch_heartbeat_recovers_after_transient_pool_timeout(
    monkeypatch,
) -> None:
    refresh_recovered = threading.Event()
    attempts = 0

    def recovering_refresh(
        leases: tuple[TaskLease, ...],
    ) -> dict[tuple[int, str, str | None], TaskLeaseRefreshState]:
        nonlocal attempts
        attempts += 1
        if attempts == 1:
            raise SQLAlchemyTimeoutError("transient pool checkout timeout")
        refresh_recovered.set()
        return {
            (lease.task_id, lease.runner_id, lease.run_id, lease.attempt_id): (
                TaskLeaseRefreshState.REFRESHED
            )
            for lease in leases
        }

    monkeypatch.setattr(
        task_lease_service,
        "refresh_task_leases_isolated",
        recovering_refresh,
    )
    monkeypatch.setattr(
        task_lease_service,
        "get_task_lease_heartbeat_seconds",
        lambda: 0.001,
    )

    stop_event = asyncio.Event()
    heartbeat_task = asyncio.create_task(
        run_task_lease_heartbeat(
            TaskLease(
                task_id=1,
                runner_id="runner-a",
                run_id="run-a",
                attempt_id="test-attempt",
            ),
            stop_event,
        )
    )
    assert await asyncio.to_thread(refresh_recovered.wait, GUARD_TIMEOUT)

    outcome = await stop_task_lease_heartbeat(heartbeat_task, stop_event)
    await task_lease_service.wait_for_heartbeat_manager_idle()

    assert outcome.requires_ttl_recovery is False


@pytest.mark.asyncio
async def test_cancellation_safe_acquire_drains_and_cleans_returned_lease() -> None:
    acquire_started = threading.Event()
    allow_acquire_to_finish = threading.Event()
    cleanup_started = threading.Event()
    allow_cleanup_to_finish = threading.Event()
    expected_lease = TaskLease(
        task_id=9, runner_id="runner-a", run_id="run-a", attempt_id="test-attempt"
    )
    cleaned_leases: list[TaskLease] = []

    def acquire() -> TaskLease:
        acquire_started.set()
        assert allow_acquire_to_finish.wait(timeout=GUARD_TIMEOUT)
        return expected_lease

    def cleanup(lease: TaskLease) -> None:
        cleanup_started.set()
        assert allow_cleanup_to_finish.wait(timeout=GUARD_TIMEOUT)
        cleaned_leases.append(lease)

    operation = asyncio.create_task(
        task_lease_service.acquire_task_lease_cancellation_safe(acquire, cleanup)
    )
    assert await asyncio.to_thread(acquire_started.wait, GUARD_TIMEOUT)
    operation.cancel()
    await asyncio.sleep(0.02)
    assert not operation.done()

    allow_acquire_to_finish.set()
    assert await asyncio.to_thread(cleanup_started.wait, GUARD_TIMEOUT)
    assert not operation.done()

    allow_cleanup_to_finish.set()
    with pytest.raises(asyncio.CancelledError):
        await asyncio.wait_for(operation, timeout=GUARD_TIMEOUT)
    assert cleaned_leases == [expected_lease]


_CASE_WHEN_PATTERN = re.compile(r"WHEN (.+?) THEN")


def _case_when_clause(case_expr) -> str:
    """The compiled WHEN condition of a two-branch case(), with literal
    binds inlined so two structurally-identical predicates compile to the
    same string regardless of bind-parameter naming.
    """
    compiled = str(case_expr.compile(compile_kwargs={"literal_binds": True}))
    match = _CASE_WHEN_PATTERN.search(compiled)
    assert match is not None, compiled
    return match.group(1)


def test_lease_checkpoint_pointer_case_predicates_match() -> None:
    """lease_checkpoint_event_id_case and lease_checkpoint_trace_event_id_case
    must clear their column under exactly the same condition.

    Both builders share _checkpoint_pointer_clearing_predicate(), so this
    passes by construction today; the assertion is what actually enforces
    it -- a future edit that inlines a diverging predicate into one builder
    would otherwise pass every other test here (each builder's own column
    still clears/retains correctly in isolation) while silently breaking
    the recovery CAS fence's two-column conjunction, which depends on both
    columns clearing together (see recover_expired_task_lease_no_commit).
    """
    legacy_when = _case_when_clause(task_lease_service.lease_checkpoint_event_id_case())
    exact_when = _case_when_clause(
        task_lease_service.lease_checkpoint_trace_event_id_case()
    )
    assert legacy_when == exact_when


def test_new_run_lease_claim_rejects_a_second_claim(db_session) -> None:
    task = _create_task(db_session)
    task.last_checkpoint_event_id = "previous-run-checkpoint"
    previous_checkpoint = TraceEvent(
        task_id=task.id,
        event_id="previous-run-checkpoint",
        event_type="system_update_general",
        timestamp=utc_now(),
        data={"checkpoint_type": CHECKPOINT_TYPE, "snapshot": {"type": "checkpoint"}},
    )
    db_session.add(previous_checkpoint)
    db_session.flush()
    task.last_checkpoint_trace_event_id = previous_checkpoint.id
    db_session.commit()

    first = acquire_task_lease(
        db_session,
        int(task.id),
        runner_id="runner-a",
        new_run=True,
    )
    second = acquire_task_lease(
        db_session,
        int(task.id),
        runner_id="runner-a",
        new_run=True,
    )

    assert first is not None
    assert first.run_id
    assert second is None
    db_session.refresh(task)
    assert task.run_id == first.run_id
    assert task.last_checkpoint_event_id is None
    # A new run clears both pointer columns together -- see
    # lease_checkpoint_trace_event_id_case's docstring for why a single
    # column left behind would desync the recovery CAS fence.
    assert task.last_checkpoint_trace_event_id is None


def test_same_run_resume_lease_preserves_checkpoint_pointer(db_session) -> None:
    task = _create_task(db_session, status=TaskStatus.WAITING_FOR_USER)
    task.run_id = "resumable-run"
    task.last_checkpoint_event_id = "current-run-checkpoint"
    current_checkpoint = TraceEvent(
        task_id=task.id,
        event_id="current-run-checkpoint",
        event_type="system_update_general",
        timestamp=utc_now(),
        data={"checkpoint_type": CHECKPOINT_TYPE, "snapshot": {"type": "checkpoint"}},
    )
    db_session.add(current_checkpoint)
    db_session.flush()
    task.last_checkpoint_trace_event_id = current_checkpoint.id
    db_session.commit()

    lease = acquire_task_lease(
        db_session,
        int(task.id),
        runner_id="resume-runner",
        expected_run_id="resumable-run",
    )
    assert lease is not None

    assert lease == TaskLease(
        task_id=int(task.id),
        runner_id="resume-runner",
        run_id="resumable-run",
        # Not part of what this test pins: a fresh attempt id per claim is
        # acquire's contract, exercised by the stamping tests below.
        attempt_id=lease.attempt_id,
    )
    db_session.refresh(task)
    assert task.last_checkpoint_event_id == "current-run-checkpoint"
    # A same-run resume retains both pointer columns together.
    assert task.last_checkpoint_trace_event_id == current_checkpoint.id


def test_running_task_reacquire_retains_checkpoint_pointer_via_case(
    db_session,
) -> None:
    """acquire_task_lease_no_commit's case()-retain branch (no new_run, no
    expected_run_id) only fires when the row is already a live RUNNING row
    with a run id -- exercised here by a heartbeat-style reacquire on a
    task that is already RUNNING with the same runner, unlike every other
    acquire_task_lease call in this file, which starts from a non-RUNNING
    task and only ever hits the case()'s clear side.
    """
    task = _create_task(db_session, status=TaskStatus.RUNNING)
    task.runner_id = "runner-a"
    task.lease_attempt_id = "test-attempt"
    task.run_id = "existing-run"
    task.last_checkpoint_event_id = "existing-checkpoint"
    checkpoint = TraceEvent(
        task_id=task.id,
        event_id="existing-checkpoint",
        event_type="system_update_general",
        timestamp=utc_now(),
        data={"checkpoint_type": CHECKPOINT_TYPE, "snapshot": {"type": "checkpoint"}},
    )
    db_session.add(checkpoint)
    db_session.flush()
    task.last_checkpoint_trace_event_id = checkpoint.id
    db_session.commit()

    lease = acquire_task_lease(db_session, int(task.id), runner_id="runner-a")

    assert lease is not None
    assert lease.run_id == "existing-run"
    db_session.refresh(task)
    assert task.last_checkpoint_event_id == "existing-checkpoint"
    # Both pointer columns retained together via the paired case() builders.
    assert task.last_checkpoint_trace_event_id == checkpoint.id


def test_lease_acquire_rejects_a_superseded_run(db_session) -> None:
    task = _create_task(db_session, status=TaskStatus.RUNNING)
    task.run_id = "current-run"
    task.control_state = "running"
    task.state_version = 3
    db_session.commit()

    assert (
        acquire_task_lease(
            db_session,
            int(task.id),
            runner_id="runner-a",
            expected_run_id="old-run",
        )
        is None
    )
    db_session.refresh(task)
    assert task.run_id == "current-run"
    assert task.state_version == 3


def test_old_lease_cannot_refresh_or_release_a_new_run(db_session) -> None:
    task = _create_task(db_session)
    old_lease = acquire_task_lease(db_session, int(task.id), runner_id="runner-a")
    assert old_lease is not None

    task.run_id = "new-run"
    task.status = TaskStatus.RUNNING
    task.control_state = "running"
    task.runner_id = "runner-a"
    task.lease_attempt_id = "test-attempt"
    db_session.commit()

    assert refresh_task_lease(db_session, old_lease) == TaskLeaseRefreshState.LOST
    assert release_task_lease(db_session, old_lease, status=TaskStatus.FAILED) is False
    db_session.refresh(task)
    assert task.run_id == "new-run"
    assert task.status == TaskStatus.RUNNING


def test_stale_running_task_with_checkpoint_becomes_paused(db_session) -> None:
    task = _create_task(db_session, status=TaskStatus.RUNNING)
    task.runner_id = "dead-runner"
    task.run_id = "checkpoint-run"
    task.lease_attempt_id = "stale-attempt"
    task.lease_expires_at = utc_now() - timedelta(seconds=1)
    task.last_checkpoint_event_id = "checkpoint-1"
    checkpoint_event = TraceEvent(
        task_id=task.id,
        event_id="checkpoint-1",
        event_type="system_update_general",
        timestamp=utc_now(),
        step_id=None,
        parent_event_id=None,
        data={
            "checkpoint_type": CHECKPOINT_TYPE,
            "snapshot": {"type": "checkpoint"},
            TASK_RUN_ID_TRACE_FIELD: "checkpoint-run",
        },
    )
    db_session.add(checkpoint_event)
    db_session.flush()
    # Exercises the PK-anchored resolution path (not just the legacy
    # string fallback): recovery must reach the same PAUSED verdict.
    task.last_checkpoint_trace_event_id = checkpoint_event.id
    db_session.commit()

    assert _recover_expired_task(db_session, task) == TaskStatus.PAUSED
    db_session.refresh(task)
    assert task.status == TaskStatus.PAUSED
    assert task.runner_id is None
    assert task.lease_expires_at is None
    assert task.lease_attempt_id is None


def test_stale_running_task_ignores_child_agent_checkpoint(db_session) -> None:
    task = _create_task(db_session, status=TaskStatus.RUNNING)
    task.runner_id = "dead-runner"
    task.lease_attempt_id = "test-attempt"
    task.run_id = "child-checkpoint-run"
    task.lease_expires_at = utc_now() - timedelta(seconds=1)
    task.last_checkpoint_event_id = "child-checkpoint-1"
    child_checkpoint_event = TraceEvent(
        task_id=task.id,
        build_id="agent_123_child",
        event_id="child-checkpoint-1",
        event_type="system_update_general",
        timestamp=utc_now(),
        step_id=None,
        parent_event_id=None,
        data={
            "checkpoint_type": CHECKPOINT_TYPE,
            "snapshot": {"type": "checkpoint"},
            TASK_RUN_ID_TRACE_FIELD: "child-checkpoint-run",
        },
    )
    db_session.add(child_checkpoint_event)
    db_session.flush()
    # Exercises the PK-anchored resolution path: an anchored build-scoped
    # row must fail its own validation (build_id is not None), the same
    # FAILED verdict the legacy scan reaches -- and it must not fall back
    # to searching other rows for a match.
    task.last_checkpoint_trace_event_id = child_checkpoint_event.id
    db_session.commit()

    assert _recover_expired_task(db_session, task) == TaskStatus.FAILED
    db_session.refresh(task)
    assert task.status == TaskStatus.FAILED
    assert task.runner_id is None
    assert task.lease_expires_at is None


def test_stale_running_task_with_legacy_checkpoint_becomes_paused(db_session) -> None:
    task = _create_task(db_session, status=TaskStatus.RUNNING)
    task.runner_id = "dead-runner"
    task.lease_attempt_id = "test-attempt"
    task.run_id = "legacy-checkpoint-run"
    task.lease_expires_at = utc_now() - timedelta(seconds=1)
    task.last_checkpoint_event_id = "legacy-checkpoint-1"
    legacy_checkpoint_event = TraceEvent(
        task_id=task.id,
        event_id="legacy-checkpoint-1",
        event_type="system_update_general",
        timestamp=utc_now(),
        step_id=None,
        parent_event_id=None,
        data={
            "checkpoint_type": next(iter(LEGACY_CHECKPOINT_TYPES)),
            "snapshot": {"type": "checkpoint"},
            TASK_RUN_ID_TRACE_FIELD: "legacy-checkpoint-run",
        },
    )
    db_session.add(legacy_checkpoint_event)
    db_session.flush()
    task.last_checkpoint_trace_event_id = legacy_checkpoint_event.id
    db_session.commit()

    assert _recover_expired_task(db_session, task) == TaskStatus.PAUSED
    db_session.refresh(task)
    assert task.status == TaskStatus.PAUSED
    assert task.runner_id is None
    assert task.lease_expires_at is None


def test_acquire_stamps_a_fresh_attempt_id_onto_the_task_row(db_session) -> None:
    """acquire -> tasks.lease_attempt_id -> TaskLease.attempt_id round-trips.

    The lease is not stamped from RETURNING (see the values-dict read-back in
    acquire_task_lease_no_commit), so the only thing proving the returned
    identity is the one actually persisted is this three-way comparison.
    """
    task = _create_task(db_session, status=TaskStatus.PENDING)

    lease = acquire_task_lease(db_session, int(task.id), runner_id="runner-a")

    assert lease is not None
    assert lease.attempt_id is not None
    # Not a placeholder: it must parse as a uuid and fit String(64).
    uuid.UUID(lease.attempt_id)
    assert len(lease.attempt_id) <= 64

    db_session.refresh(task)
    assert task.lease_attempt_id == lease.attempt_id


def test_each_acquisition_mints_a_new_attempt_id(db_session) -> None:
    """A new value per claim -- this is what state_version cannot do."""
    task = _create_task(db_session, status=TaskStatus.PENDING)

    first = acquire_task_lease(db_session, int(task.id), runner_id="runner-a")
    assert first is not None
    release_task_lease(db_session, first, status=TaskStatus.PAUSED)

    second = acquire_task_lease(db_session, int(task.id), runner_id="runner-a")
    assert second is not None

    assert first.attempt_id != second.attempt_id
    db_session.refresh(task)
    assert task.lease_attempt_id == second.attempt_id


def test_same_run_resume_still_rotates_the_attempt_id(db_session) -> None:
    """expected_run_id keeps the run but not the attempt: this is exactly the
    shape where a successor run reacquires the SAME (runner_id, run_id)
    tuple and must still be distinguishable from the stale run holding it."""
    task = _create_task(db_session, status=TaskStatus.WAITING_FOR_USER)
    task.run_id = "resumable-run"
    db_session.commit()

    first = acquire_task_lease(
        db_session,
        int(task.id),
        runner_id="same-runner",
        expected_run_id="resumable-run",
    )
    second = acquire_task_lease(
        db_session,
        int(task.id),
        runner_id="same-runner",
        expected_run_id="resumable-run",
    )

    assert first is not None and second is not None
    assert (first.runner_id, first.run_id) == (second.runner_id, second.run_id)
    assert first.attempt_id != second.attempt_id


def test_new_run_acquisition_stamps_an_attempt_id(db_session) -> None:
    """The stamp lives in the values-dict base, not a conditional branch --
    it must fire on the new_run=True path too."""
    task = _create_task(db_session, status=TaskStatus.PENDING)

    lease = acquire_task_lease(
        db_session, int(task.id), runner_id="runner-a", new_run=True
    )

    assert lease is not None
    assert lease.attempt_id is not None
    uuid.UUID(lease.attempt_id)
    db_session.refresh(task)
    assert task.lease_attempt_id == lease.attempt_id


def test_release_clears_the_attempt_id(db_session) -> None:
    """The column must not carry a finished attempt's value."""
    task = _create_task(db_session, status=TaskStatus.PENDING)
    lease = acquire_task_lease(db_session, int(task.id), runner_id="runner-a")
    assert lease is not None
    db_session.refresh(task)
    assert task.lease_attempt_id is not None

    release_task_lease(db_session, lease, status=TaskStatus.PAUSED)

    db_session.refresh(task)
    assert task.lease_attempt_id is None


def test_expired_lease_recovery_clears_the_attempt_id(db_session) -> None:
    """Same invariant as release, exercised through expiry recovery.

    No checkpoint is set up, so recovery lands on FAILED rather than PAUSED
    -- that branch distinction is orthogonal to what this test pins.
    """
    task = _create_task(db_session, status=TaskStatus.RUNNING)
    task.runner_id = "dead-runner"
    task.run_id = "expiring-run"
    task.lease_expires_at = utc_now() - timedelta(seconds=1)
    task.lease_attempt_id = "stale-attempt"
    db_session.commit()
    assert task.lease_attempt_id is not None

    assert _recover_expired_task(db_session, task) == TaskStatus.FAILED

    db_session.refresh(task)
    assert task.lease_attempt_id is None


def test_fail_and_release_clears_the_attempt_id(db_session) -> None:
    """Same invariant, exercised through the fail-and-release writer."""
    task = _create_task(db_session)
    lease = acquire_task_lease(db_session, int(task.id), runner_id="runner-a")
    assert lease is not None
    db_session.refresh(task)
    assert task.lease_attempt_id is not None

    changed = task_lease_service.fail_and_release_task_lease_no_commit(
        db_session,
        lease,
        error_message="setup failed",
    )
    db_session.commit()

    assert changed is True
    db_session.refresh(task)
    assert task.lease_attempt_id is None


def test_release_current_runner_task_lease_clears_the_attempt_id(db_session) -> None:
    """Same invariant, exercised through the current-runner release path."""
    task = _create_task(db_session, status=TaskStatus.PENDING)
    lease = acquire_task_lease(db_session, int(task.id), runner_id="runner-a")
    assert lease is not None
    db_session.refresh(task)
    assert task.lease_attempt_id is not None

    with task_lease_service.bind_task_lease_context(lease):
        changed = task_lease_service.release_current_runner_task_lease(
            db_session,
            int(task.id),
            status=TaskStatus.PAUSED,
            runner_id="runner-a",
        )

    assert changed is True
    db_session.refresh(task)
    assert task.lease_attempt_id is None


def test_task_lease_snapshot_carries_the_routing_epoch() -> None:
    """A routing key includes the epoch; it must resolve to a local holder."""
    from types import SimpleNamespace

    from xagent.web.services.task_command_execution import _task_lease_snapshot

    row = SimpleNamespace(
        id=7,
        runner_id="runner-a",
        run_id="run-b",
        lease_attempt_id="attempt-c",
    )
    lease = _task_lease_snapshot(row)
    assert lease is not None
    assert lease.attempt_id == "attempt-c"