Skip to content

Enterprise agent platform for building, deploying, and operating AI teammates.

37.5 KB
Raw Blame
from __future__ import annotations

import asyncio
import threading
import time
from concurrent.futures import ThreadPoolExecutor
from dataclasses import replace
from datetime import datetime, timedelta, timezone

import pytest
import sqlalchemy as sa
from sqlalchemy.orm import sessionmaker

from xagent.sandbox.base import ExactGenerationProbe
from xagent.web.models.sandbox import DurableSandboxLifecycle
from xagent.web.services.durable_sandbox_lifecycle import (
    DurableLifecycleConflict,
    DurableSandboxDeleteCoordinator,
    DurableSandboxLifecycleRepository,
    DurableSandboxLifecycleService,
    LifecycleFence,
    RegisterLifecycle,
    backend_lifecycle_digest,
    digest_turn_identifier,
)

NOW = datetime(2026, 9, 14, 12, tzinfo=timezone.utc)


def _schema(engine: sa.Engine) -> None:
    metadata = sa.MetaData()
    sa.Table(
        "tasks",
        metadata,
        sa.Column("id", sa.Integer(), primary_key=True),
        sa.Column("runner_id", sa.String(255)),
        sa.Column("run_id", sa.String(64)),
        sa.Column("lease_attempt_id", sa.String(64)),
        sa.Column("status", sa.String(32)),
        sa.Column("lease_expires_at", sa.DateTime(timezone=True)),
    )
    metadata.create_all(engine)
    DurableSandboxLifecycle.__table__.create(engine)


@pytest.fixture
def sessions(tmp_path):
    engine = sa.create_engine(
        f"sqlite:///{tmp_path / 'lifecycles.db'}",
        connect_args={"check_same_thread": False, "timeout": 10},
    )
    with engine.begin() as connection:
        connection.exec_driver_sql("PRAGMA journal_mode=WAL")
    _schema(engine)
    factory = sessionmaker(bind=engine, expire_on_commit=False)
    try:
        yield factory
    finally:
        engine.dispose()


def _request(scope: str = "a" * 64) -> RegisterLifecycle:
    return RegisterLifecycle(
        scope_digest=scope,
        task_id=7,
        run_id="run-1",
        lease_attempt_id="attempt-1",
        turn_digest=digest_turn_identifier("raw-turn-1"),
        eligible_at=NOW - timedelta(minutes=2),
        owner_lease_expires_at=NOW - timedelta(minutes=1),
    )


def _set_task(db, *, active: bool, attempt: str = "attempt-1") -> None:
    db.execute(sa.text("DELETE FROM tasks"))
    if active:
        db.execute(
            sa.text(
                "INSERT INTO tasks "
                "(id, runner_id, run_id, lease_attempt_id, status, lease_expires_at) "
                "VALUES (:id, :runner, :run, :attempt, :status, :expiry)"
            ),
            {
                "id": 7,
                "runner": "runner-1",
                "run": "run-1",
                "attempt": attempt,
                "status": "RUNNING",
                "expiry": NOW + timedelta(minutes=1),
            },
        )


def test_register_is_durable_before_create_and_contains_only_opaque_identity(
    sessions,
) -> None:
    with sessions() as db:
        fence = DurableSandboxLifecycleRepository(db).register(_request())
        db.commit()

    assert fence.state == "registered"
    assert fence.active_scope_digest == fence.scope_digest
    assert fence.create_phase == "not_started"
    assert fence.create_terminal_outcome is None
    assert len(fence.create_operation_token) == 64
    assert fence.version == 1
    assert fence.backend_lifecycle_digest == backend_lifecycle_digest(
        fence.scope_digest, fence.lifecycle_token
    )
    assert fence.backend_lifecycle_digest not in {
        fence.scope_digest,
        fence.lifecycle_token,
    }
    columns = {column.name for column in DurableSandboxLifecycle.__table__.columns}
    forbidden = {
        "resource_owner_key",
        "user",
        "user_id",
        "credentials",
        "env",
        "actor_id",
        "session_identity",
        "turn_id",
    }
    assert columns.isdisjoint(forbidden)
    with sessions() as db:
        row = db.execute(sa.select(DurableSandboxLifecycle)).scalar_one()
        persisted = " ".join(str(value) for value in row.__dict__.values())
    assert "raw-turn-1" not in persisted


def test_duplicate_scope_registration_fails_closed(sessions) -> None:
    with sessions() as db:
        DurableSandboxLifecycleRepository(db).register(_request())
        db.commit()
    with sessions() as db:
        with pytest.raises(DurableLifecycleConflict):
            DurableSandboxLifecycleRepository(db).register(_request())


def test_service_commits_registration_before_returning(sessions) -> None:
    service = DurableSandboxLifecycleService(sessions)
    fence = service.register(_request())
    with sessions() as independent_session:
        persisted = DurableSandboxLifecycleRepository(independent_session).get_by_scope(
            fence.scope_digest
        )
    assert persisted is not None
    assert persisted.lifecycle_token == fence.lifecycle_token


def test_ready_and_renew_require_the_exact_active_attempt(sessions) -> None:
    with sessions() as db:
        repo = DurableSandboxLifecycleRepository(db)
        registered = repo.register(_request())
        _set_task(db, active=True, attempt="successor-attempt")
        assert (
            repo.mark_ready(
                registered,
                now=NOW,
                owner_lease_expires_at=NOW + timedelta(minutes=1),
            )
            is None
        )
        _set_task(db, active=True)
        begun = repo.begin_create(registered, now=NOW)
        assert begun is not None
        completed = repo.complete_create(begun, now=NOW, outcome="success")
        assert completed is not None
        ready = repo.mark_ready(
            completed,
            now=NOW,
            owner_lease_expires_at=NOW + timedelta(minutes=1),
        )
        assert ready is not None
        assert ready.state == "ready"
        assert ready.version == completed.version + 1
        _set_task(db, active=True, attempt="successor-attempt")
        assert (
            repo.renew(
                ready,
                now=NOW,
                owner_lease_expires_at=NOW + timedelta(minutes=2),
            )
            is None
        )


def test_begin_and_complete_create_are_exact_monotonic_cas_transitions(
    sessions,
) -> None:
    with sessions() as db:
        repo = DurableSandboxLifecycleRepository(db)
        registered = repo.register(_request())
        _set_task(db, active=True, attempt="successor-attempt")
        assert repo.begin_create(registered, now=NOW) is None
        _set_task(db, active=True)
        begun = repo.begin_create(registered, now=NOW)
        assert begun is not None
        assert begun.create_phase == "may_publish"
        assert begun.version == registered.version + 1
        assert repo.begin_create(registered, now=NOW) is None

        wrong_operation = LifecycleFence(
            **{
                **begun.__dict__,
                "create_operation_token": "f" * 64,
            }
        )
        assert repo.complete_create(wrong_operation, now=NOW, outcome="success") is None
        completed = repo.complete_create(begun, now=NOW, outcome="success")
        assert completed is not None
        assert completed.create_phase == "terminal"
        assert completed.create_terminal_outcome == "success"
        assert repo.complete_create(begun, now=NOW, outcome="success") is None


def test_ready_requires_terminal_create_phase(sessions) -> None:
    with sessions() as db:
        repo = DurableSandboxLifecycleRepository(db)
        registered = repo.register(_request())
        _set_task(db, active=True)
        assert (
            repo.mark_ready(
                registered,
                now=NOW,
                owner_lease_expires_at=NOW + timedelta(minutes=1),
            )
            is None
        )


def test_terminal_absent_create_cannot_be_marked_ready(sessions) -> None:
    with sessions() as db:
        repo = DurableSandboxLifecycleRepository(db)
        registered = repo.register(_request())
        _set_task(db, active=True)
        begun = repo.begin_create(registered, now=NOW)
        assert begun is not None
        absent = repo.complete_create(begun, now=NOW, outcome="terminal_absent")
        assert absent is not None
        assert absent.create_terminal_outcome == "terminal_absent"
        assert (
            repo.mark_ready(
                absent,
                now=NOW,
                owner_lease_expires_at=NOW + timedelta(minutes=1),
            )
            is None
        )


def test_create_phase_and_state_guards_reject_current_version_fences(sessions) -> None:
    with sessions() as db:
        repo = DurableSandboxLifecycleRepository(db)
        registered = repo.register(_request())
        _set_task(db, active=True)

        # Removing quarantine_create's may_publish guard makes this succeed.
        assert (
            repo.quarantine_create(registered, now=NOW, claim_ttl=timedelta(minutes=1))
            is None
        )

        begun = repo.begin_create(registered, now=NOW)
        assert begun is not None
        db.execute(
            sa.update(DurableSandboxLifecycle)
            .where(DurableSandboxLifecycle.id == begun.id)
            .values(
                state="deleting",
                active_scope_digest=None,
                deleting_at=NOW,
                delete_claim_expires_at=NOW + timedelta(minutes=1),
            )
        )
        # The fence still has the current owner/version; only state rejects it.
        assert repo.complete_create(begun, now=NOW, outcome="success") is None

        db.execute(
            sa.update(DurableSandboxLifecycle)
            .where(DurableSandboxLifecycle.id == begun.id)
            .values(
                create_phase="terminal",
                create_terminal_outcome="success",
            )
        )
        current = repo.list_by_scope(begun.scope_digest)[0]
        # Removing observe_create's may_publish guard makes this succeed.
        assert repo.observe_create(current, now=NOW) is None


def test_age_only_never_reclaims_an_exact_active_attempt(sessions) -> None:
    with sessions() as db:
        repo = DurableSandboxLifecycleRepository(db)
        fence = repo.register(_request())
        _set_task(db, active=True)
        db.commit()
    with sessions() as db:
        claimed = DurableSandboxLifecycleRepository(db).claim_for_delete(
            fence, now=NOW, claim_ttl=timedelta(seconds=30)
        )
        assert claimed is None


def test_lost_attempt_can_be_claimed_and_tombstone_is_retryable(sessions) -> None:
    with sessions() as db:
        repo = DurableSandboxLifecycleRepository(db)
        registered = repo.register(_request())
        _set_task(db, active=True, attempt="successor-attempt")
        claimed = repo.claim_for_delete(
            registered, now=NOW, claim_ttl=timedelta(seconds=30)
        )
        assert claimed is not None
        assert claimed.state == "deleting"
        assert claimed.owner_token != registered.owner_token
        assert claimed.version == registered.version + 1
        assert claimed.delete_attempts == 1
        backed_off = repo.backoff_delete(
            claimed, now=NOW, retry_at=NOW + timedelta(minutes=1)
        )
        assert backed_off is not None
        assert backed_off.version == claimed.version + 1
        db.commit()

    with sessions() as db:
        repo = DurableSandboxLifecycleRepository(db)
        assert (
            repo.reclaim_delete(
                backed_off,
                now=NOW + timedelta(seconds=45),
                claim_ttl=timedelta(seconds=30),
            )
            is None
        )
        reclaimed = repo.reclaim_delete(
            backed_off,
            now=NOW + timedelta(minutes=2),
            claim_ttl=timedelta(seconds=30),
        )
        assert reclaimed is not None
        assert reclaimed.owner_token != backed_off.owner_token
        assert reclaimed.version == backed_off.version + 1
        assert reclaimed.delete_attempts == 2


def test_active_owner_can_tombstone_ready_terminal_generation(sessions) -> None:
    with sessions() as db:
        repo = DurableSandboxLifecycleRepository(db)
        registered = repo.register(_request())
        _set_task(db, active=True)
        begun = repo.begin_create(registered, now=NOW)
        assert begun is not None
        completed = repo.complete_create(begun, now=NOW, outcome="success")
        assert completed is not None
        ready = repo.mark_ready(
            completed,
            now=NOW,
            owner_lease_expires_at=NOW + timedelta(minutes=1),
        )
        assert ready is not None

        deleting = repo.mark_deleting_owned(
            ready, now=NOW, claim_ttl=timedelta(minutes=1)
        )

        assert deleting is not None
        assert deleting.state == "deleting"
        assert deleting.create_phase == "terminal"
        assert deleting.active_scope_digest is None
        assert deleting.owner_token != ready.owner_token
        assert deleting.version == ready.version + 1
        assert deleting.delete_attempts == 1


def test_active_owner_can_tombstone_registered_terminal_attach_failure(
    sessions,
) -> None:
    with sessions() as db:
        repo = DurableSandboxLifecycleRepository(db)
        registered = repo.register(_request())
        _set_task(db, active=True)
        begun = repo.begin_create(registered, now=NOW)
        assert begun is not None
        completed = repo.complete_create(begun, now=NOW, outcome="success")
        assert completed is not None

        deleting = repo.mark_deleting_owned(
            completed, now=NOW, claim_ttl=timedelta(minutes=1)
        )

        assert deleting is not None
        assert deleting.state == "deleting"
        assert deleting.create_phase == "terminal"


@pytest.mark.parametrize("phase", ["not_started", "observed"])
def test_active_owner_can_tombstone_other_non_ambiguous_phases(sessions, phase) -> None:
    with sessions() as db:
        repo = DurableSandboxLifecycleRepository(db)
        registered = repo.register(_request())
        if phase == "observed":
            db.execute(
                sa.update(DurableSandboxLifecycle)
                .where(DurableSandboxLifecycle.id == registered.id)
                .values(create_phase="observed")
            )

        deleting = repo.mark_deleting_owned(
            registered, now=NOW, claim_ttl=timedelta(minutes=1)
        )

        assert deleting is not None
        assert deleting.state == "deleting"
        assert deleting.create_phase == phase


def test_owned_tombstone_rejects_ambiguous_and_stale_fences(sessions) -> None:
    with sessions() as db:
        repo = DurableSandboxLifecycleRepository(db)
        registered = repo.register(_request())
        _set_task(db, active=True)
        begun = repo.begin_create(registered, now=NOW)
        assert begun is not None
        assert (
            repo.mark_deleting_owned(begun, now=NOW, claim_ttl=timedelta(minutes=1))
            is None
        )

        terminal = repo.complete_create(begun, now=NOW, outcome="success")
        assert terminal is not None
        corruptions = (
            replace(terminal, owner_token="f" * 64),
            replace(terminal, version=terminal.version + 1),
            replace(terminal, create_operation_token="e" * 64),
            replace(terminal, lifecycle_token="d" * 64),
            replace(terminal, backend_lifecycle_digest="c" * 64),
        )
        for stale in corruptions:
            assert (
                repo.mark_deleting_owned(stale, now=NOW, claim_ttl=timedelta(minutes=1))
                is None
            )

        deleting = repo.mark_deleting_owned(
            terminal, now=NOW, claim_ttl=timedelta(minutes=1)
        )
        assert deleting is not None
        assert (
            repo.mark_deleting_owned(deleting, now=NOW, claim_ttl=timedelta(minutes=1))
            is None
        )


def test_two_real_sessions_allow_only_one_delete_claim(sessions) -> None:
    with sessions() as db:
        fence = DurableSandboxLifecycleRepository(db).register(_request())
        db.commit()

    barrier = threading.Barrier(2)

    def claim():
        with sessions() as db:
            barrier.wait(timeout=5)
            result = DurableSandboxLifecycleRepository(db).claim_for_delete(
                fence, now=NOW, claim_ttl=timedelta(minutes=1)
            )
            db.commit()
            return result

    with ThreadPoolExecutor(max_workers=2) as executor:
        outcomes = list(executor.map(lambda _: claim(), range(2)))
    winners = [outcome for outcome in outcomes if outcome is not None]
    assert len(winners) == 1
    assert winners[0].delete_attempts == 1


def _race_transition(sessions, transition):
    barrier = threading.Barrier(2)

    def run():
        with sessions() as db:
            barrier.wait(timeout=5)
            result = transition(DurableSandboxLifecycleRepository(db))
            db.commit()
            return result

    with ThreadPoolExecutor(max_workers=2) as executor:
        return list(executor.map(lambda _: run(), range(2)))


def test_two_real_sessions_allow_only_one_begin_create(sessions) -> None:
    with sessions() as db:
        fence = DurableSandboxLifecycleRepository(db).register(_request())
        _set_task(db, active=True)
        db.commit()
    outcomes = _race_transition(
        sessions, lambda repo: repo.begin_create(fence, now=NOW)
    )
    assert sum(outcome is not None for outcome in outcomes) == 1


def test_two_real_sessions_allow_only_one_complete_create(sessions) -> None:
    with sessions() as db:
        repo = DurableSandboxLifecycleRepository(db)
        fence = repo.register(_request())
        _set_task(db, active=True)
        begun = repo.begin_create(fence, now=NOW)
        assert begun is not None
        db.commit()
    outcomes = _race_transition(
        sessions,
        lambda repo: repo.complete_create(begun, now=NOW, outcome="success"),
    )
    assert sum(outcome is not None for outcome in outcomes) == 1


def test_two_real_sessions_allow_only_one_observe_create(sessions) -> None:
    service, claimed = _claimed_may_publish(sessions)
    outcomes = _race_transition(
        sessions, lambda repo: repo.observe_create(claimed, now=NOW)
    )
    assert sum(outcome is not None for outcome in outcomes) == 1


def test_two_real_sessions_allow_only_one_owner_quarantine(sessions) -> None:
    with sessions() as db:
        repo = DurableSandboxLifecycleRepository(db)
        registered = repo.register(_request())
        _set_task(db, active=True)
        begun = repo.begin_create(registered, now=NOW)
        assert begun is not None
        db.commit()
    outcomes = _race_transition(
        sessions,
        lambda repo: repo.quarantine_create(
            begun, now=NOW, claim_ttl=timedelta(minutes=1)
        ),
    )
    assert sum(outcome is not None for outcome in outcomes) == 1


@pytest.mark.parametrize("outcome", ["success", "terminal_absent"])
def test_late_completion_monotonically_completes_quarantined_create(
    sessions, outcome
) -> None:
    service, quarantined = _claimed_may_publish(sessions)
    stale_create_fence = replace(
        quarantined,
        owner_token="f" * 64,
        version=1,
    )

    completed = service.complete_quarantined_create(
        stale_create_fence, now=NOW, outcome=outcome
    )

    assert completed is not None
    assert completed.state == "deleting"
    assert completed.create_phase == "terminal"
    assert completed.owner_token == quarantined.owner_token
    assert completed.version == quarantined.version + 1
    assert (
        service.complete_quarantined_create(
            stale_create_fence, now=NOW, outcome=outcome
        )
        is None
    )


def test_quarantined_completion_requires_exact_operation_generation_and_phase(
    sessions,
) -> None:
    service = DurableSandboxLifecycleService(sessions)
    registered = service.register(_request())
    with sessions() as db:
        _set_task(db, active=True)
        db.commit()
    begun = service.begin_create(registered, now=NOW)
    assert begun is not None

    assert (
        service.complete_quarantined_create(begun, now=NOW, outcome="success") is None
    )
    quarantined = service.quarantine_create(
        begun, now=NOW, claim_ttl=timedelta(minutes=1)
    )
    assert quarantined is not None
    corruptions = (
        replace(begun, create_operation_token="f" * 64),
        replace(begun, lifecycle_token="e" * 64),
        replace(begun, backend_lifecycle_digest="d" * 64),
    )
    for stale in corruptions:
        assert (
            service.complete_quarantined_create(stale, now=NOW, outcome="success")
            is None
        )

    observed = service.observe_create(quarantined, now=NOW)
    assert observed is not None
    assert (
        service.complete_quarantined_create(begun, now=NOW, outcome="success") is None
    )
    with pytest.raises(ValueError, match="unsupported create completion outcome"):
        service.complete_quarantined_create(  # type: ignore[arg-type]
            begun, now=NOW, outcome="unknown"
        )


def test_late_completion_invalidates_sweeper_fence_and_returns_current_claim(
    sessions,
) -> None:
    service, quarantined = _claimed_may_publish(sessions)

    completed = service.complete_quarantined_create(
        quarantined, now=NOW, outcome="success"
    )

    assert completed is not None
    assert (
        service.backoff_delete(
            quarantined,
            now=NOW,
            retry_at=NOW + timedelta(minutes=1),
        )
        is None
    )
    current = service.list_by_scope(quarantined.scope_digest)[0]
    assert current == completed
    backed_off = service.backoff_delete(
        current,
        now=NOW,
        retry_at=NOW + timedelta(minutes=1),
    )
    assert backed_off is not None


def test_late_completion_racing_sweeper_leaves_only_current_terminal_fence(
    sessions,
) -> None:
    service, quarantined = _claimed_may_publish(sessions)
    barrier = threading.Barrier(2)

    def complete():
        with sessions() as db:
            barrier.wait(timeout=5)
            result = DurableSandboxLifecycleRepository(db).complete_quarantined_create(
                quarantined, now=NOW, outcome="success"
            )
            db.commit()
            return result

    def backoff():
        with sessions() as db:
            barrier.wait(timeout=5)
            result = DurableSandboxLifecycleRepository(db).backoff_delete(
                quarantined,
                now=NOW,
                retry_at=NOW + timedelta(minutes=1),
            )
            db.commit()
            return result

    with ThreadPoolExecutor(max_workers=2) as executor:
        completion_future = executor.submit(complete)
        backoff_future = executor.submit(backoff)
        completed = completion_future.result(timeout=10)
        swept = backoff_future.result(timeout=10)

    assert completed is not None
    current = service.list_by_scope(quarantined.scope_digest)[0]
    assert current.create_phase == "terminal"
    assert current.version >= completed.version
    assert (
        service.backoff_delete(
            quarantined,
            now=NOW,
            retry_at=NOW + timedelta(minutes=2),
        )
        is None
    )
    if swept is not None:
        assert (
            service.backoff_delete(
                swept,
                now=NOW,
                retry_at=NOW + timedelta(minutes=2),
            )
            is None
        )


def test_old_late_completion_cannot_touch_successor_generation(sessions) -> None:
    service, quarantined = _claimed_may_publish(sessions)
    successor = service.register(_request(quarantined.scope_digest))

    completed = service.complete_quarantined_create(
        quarantined, now=NOW, outcome="terminal_absent"
    )

    assert completed is not None
    generations = service.list_by_scope(quarantined.scope_digest)
    assert generations[0].id == quarantined.id
    assert generations[0].create_phase == "terminal"
    assert generations[1].id == successor.id
    assert generations[1].lifecycle_token == successor.lifecycle_token
    assert generations[1].create_operation_token == successor.create_operation_token
    assert generations[1].state == "registered"
    assert generations[1].create_phase == "not_started"


def test_final_claim_dml_rechecks_a_racing_successor_attempt(sessions) -> None:
    with sessions() as db:
        repo = DurableSandboxLifecycleRepository(db)
        fence = repo.register(_request())
        assert [item.id for item in repo.list_reclaimable(now=NOW)] == [fence.id]
        db.commit()
    with sessions() as db:
        _set_task(db, active=True, attempt="attempt-1")
        db.commit()
    with sessions() as db:
        assert (
            DurableSandboxLifecycleRepository(db).claim_for_delete(
                fence, now=NOW, claim_ttl=timedelta(minutes=1)
            )
            is None
        )


def test_generation_digest_prevents_stale_deleter_aba(sessions) -> None:
    with sessions() as db:
        repo = DurableSandboxLifecycleRepository(db)
        first = repo.register(_request())
        first_claim = repo.claim_for_delete(
            first, now=NOW, claim_ttl=timedelta(minutes=1)
        )
        assert first_claim is not None
        assert first_claim.active_scope_digest is None
        second = repo.register(_request())
        db.commit()

    assert second.backend_lifecycle_digest != first.backend_lifecycle_digest
    assert second.lifecycle_token != first.lifecycle_token
    with sessions() as db:
        repo = DurableSandboxLifecycleRepository(db)
        assert repo.settle_delete(first_claim)
        assert not repo.settle_delete(first_claim)
        survivor = repo.get_by_scope(second.scope_digest)
        assert survivor is not None
        assert survivor.lifecycle_token == second.lifecycle_token
        assert survivor.backend_lifecycle_digest == second.backend_lifecycle_digest


def test_quarantined_may_publish_generation_does_not_block_successor(sessions) -> None:
    with sessions() as db:
        repo = DurableSandboxLifecycleRepository(db)
        first = repo.register(_request())
        _set_task(db, active=True)
        begun = repo.begin_create(first, now=NOW)
        assert begun is not None
        quarantined = repo.quarantine_create(
            begun, now=NOW, claim_ttl=timedelta(minutes=1)
        )
        assert quarantined is not None
        assert quarantined.create_phase == "may_publish"
        assert not repo.settle_delete(quarantined)
        successor = repo.register(_request())
        generations = repo.list_by_scope(first.scope_digest)
        db.commit()

    assert [item.id for item in generations] == [first.id, successor.id]
    assert generations[0].active_scope_digest is None
    assert generations[1].active_scope_digest == first.scope_digest
    assert generations[0].backend_lifecycle_digest != (
        generations[1].backend_lifecycle_digest
    )


def test_expired_quarantine_is_reclaimable_while_successor_attempt_is_active(
    sessions,
) -> None:
    with sessions() as db:
        repo = DurableSandboxLifecycleRepository(db)
        first = repo.register(_request())
        _set_task(db, active=True)
        begun = repo.begin_create(first, now=NOW)
        assert begun is not None
        quarantined = repo.quarantine_create(
            begun, now=NOW, claim_ttl=timedelta(seconds=1)
        )
        assert quarantined is not None
        successor = repo.register(_request())
        reclaimed = repo.reclaim_delete(
            quarantined,
            now=NOW + timedelta(seconds=2),
            claim_ttl=timedelta(minutes=1),
        )
        assert reclaimed is not None
        assert reclaimed.id == quarantined.id
        assert reclaimed.version == quarantined.version + 1
        active = repo.get_by_scope(successor.scope_digest)
        assert active is not None
        assert active.id == successor.id


def test_crashed_may_publish_owner_is_quarantined_by_stale_reclaimer(sessions) -> None:
    with sessions() as db:
        repo = DurableSandboxLifecycleRepository(db)
        registered = repo.register(_request())
        _set_task(db, active=True)
        begun = repo.begin_create(registered, now=NOW)
        assert begun is not None
        _set_task(db, active=False)
        claimed = repo.claim_for_delete(begun, now=NOW, claim_ttl=timedelta(minutes=1))
        assert claimed is not None
        assert claimed.create_phase == "may_publish"
        assert claimed.active_scope_digest is None


def test_two_real_sessions_allow_only_one_active_generation(sessions) -> None:
    barrier = threading.Barrier(2)

    def register():
        with sessions() as db:
            barrier.wait(timeout=5)
            try:
                result = DurableSandboxLifecycleRepository(db).register(_request())
                db.commit()
                return result
            except DurableLifecycleConflict:
                db.rollback()
                return None

    with ThreadPoolExecutor(max_workers=2) as executor:
        outcomes = list(executor.map(lambda _: register(), range(2)))
    assert sum(outcome is not None for outcome in outcomes) == 1


def test_stale_owner_or_version_cannot_settle(sessions) -> None:
    with sessions() as db:
        repo = DurableSandboxLifecycleRepository(db)
        registered = repo.register(_request())
        claim = repo.claim_for_delete(
            registered, now=NOW, claim_ttl=timedelta(seconds=1)
        )
        assert claim is not None
        db.commit()
    with sessions() as db:
        repo = DurableSandboxLifecycleRepository(db)
        reclaimed = repo.reclaim_delete(
            claim,
            now=NOW + timedelta(seconds=2),
            claim_ttl=timedelta(minutes=1),
        )
        assert reclaimed is not None
        assert not repo.settle_delete(claim)
        assert repo.settle_delete(reclaimed)


class _Backend:
    def __init__(self, presence=ExactGenerationProbe.ABSENT, *, fail_delete=False):
        self.presence = presence
        self.fail_delete = fail_delete
        self.probes: list[str] = []
        self.deletes: list[str] = []

    async def probe_durable_sandbox_strict(self, lifecycle_id: str):
        self.probes.append(lifecycle_id)
        if isinstance(self.presence, Exception):
            raise self.presence
        return self.presence

    async def delete_durable_sandbox_strict(self, lifecycle_id: str) -> None:
        self.deletes.append(lifecycle_id)
        if self.fail_delete:
            raise RuntimeError("delete uncertain")


def _claimed_may_publish(sessions):
    service = DurableSandboxLifecycleService(sessions)
    registered = service.register(_request())
    with sessions() as db:
        _set_task(db, active=True)
        db.commit()
    begun = service.begin_create(registered, now=NOW)
    assert begun is not None
    claimed = service.quarantine_create(begun, now=NOW, claim_ttl=timedelta(minutes=1))
    assert claimed is not None
    return service, claimed


@pytest.mark.parametrize(
    "presence",
    [
        ExactGenerationProbe.ABSENT,
        ExactGenerationProbe.UNKNOWN,
        RuntimeError("probe failed"),
    ],
)
def test_ambiguous_create_absent_or_unknown_only_backs_off(sessions, presence) -> None:
    service, claimed = _claimed_may_publish(sessions)
    backend = _Backend(presence)
    settled = asyncio.run(
        DurableSandboxDeleteCoordinator(service, backend).delete_claimed(
            claimed,
            now=NOW,
            retry_at=NOW + timedelta(minutes=1),
        )
    )
    assert not settled
    assert backend.deletes == []
    rows = service.list_by_scope(claimed.scope_digest)
    assert len(rows) == 1
    assert rows[0].create_phase == "may_publish"
    assert rows[0].retry_at.replace(tzinfo=timezone.utc) == NOW + timedelta(minutes=1)


def test_coordinator_logs_backend_failure_without_losing_backoff_error(
    sessions, caplog
) -> None:
    service, claimed = _claimed_may_publish(sessions)

    def fail_backoff(*_args, **_kwargs):
        raise RuntimeError("durable backoff failed")

    service.backoff_delete = fail_backoff  # type: ignore[method-assign]
    with caplog.at_level("WARNING"):
        with pytest.raises(RuntimeError, match="durable backoff failed"):
            asyncio.run(
                DurableSandboxDeleteCoordinator(
                    service, _Backend(RuntimeError("backend probe failed"))
                ).delete_claimed(
                    claimed,
                    now=NOW,
                    retry_at=NOW + timedelta(minutes=1),
                )
            )
    assert claimed.backend_lifecycle_digest in caplog.text
    assert "backend probe failed" in caplog.text


def test_coordinator_database_transition_does_not_block_event_loop(sessions) -> None:
    service, claimed = _claimed_may_publish(sessions)
    original = service.backoff_delete
    release = threading.Event()

    def blocking_backoff(*args, **kwargs):
        release.wait(timeout=1)
        return original(*args, **kwargs)

    service.backoff_delete = blocking_backoff  # type: ignore[method-assign]

    async def run() -> None:
        timer = threading.Timer(0.3, release.set)
        timer.start()
        started = time.monotonic()
        task = asyncio.create_task(
            DurableSandboxDeleteCoordinator(service, _Backend()).delete_claimed(
                claimed,
                now=NOW,
                retry_at=NOW + timedelta(minutes=1),
            )
        )
        await asyncio.sleep(0.02)
        elapsed = time.monotonic() - started
        release.set()
        assert not await task
        timer.cancel()
        assert elapsed < 0.15

    asyncio.run(run())


def test_invalid_retry_time_fails_before_backend_or_database_io(sessions) -> None:
    service, claimed = _claimed_may_publish(sessions)
    backend = _Backend(ExactGenerationProbe.PRESENT)
    with pytest.raises(ValueError, match="retry_at must be in the future"):
        asyncio.run(
            DurableSandboxDeleteCoordinator(service, backend).delete_claimed(
                claimed,
                now=NOW,
                retry_at=NOW,
            )
        )
    assert backend.probes == []
    current = service.list_by_scope(claimed.scope_digest)[0]
    assert current.version == claimed.version
    assert current.retry_at is None


def test_present_is_observed_before_delete_and_settlement(sessions) -> None:
    service, claimed = _claimed_may_publish(sessions)

    class InspectingBackend(_Backend):
        async def delete_durable_sandbox_strict(self, lifecycle_id: str) -> None:
            row = service.list_by_scope(claimed.scope_digest)[0]
            assert row.create_phase == "observed"
            await super().delete_durable_sandbox_strict(lifecycle_id)

    backend = InspectingBackend(ExactGenerationProbe.PRESENT)
    settled = asyncio.run(
        DurableSandboxDeleteCoordinator(service, backend).delete_claimed(
            claimed,
            now=NOW,
            retry_at=NOW + timedelta(minutes=1),
        )
    )
    assert settled
    assert backend.probes == [claimed.backend_lifecycle_digest]
    assert backend.deletes == [claimed.backend_lifecycle_digest]
    assert service.list_by_scope(claimed.scope_digest) == []


def test_crash_after_observe_retries_delete_without_reprobe(sessions) -> None:
    service, claimed = _claimed_may_publish(sessions)
    observed = service.observe_create(claimed, now=NOW)
    assert observed is not None
    backend = _Backend(ExactGenerationProbe.UNKNOWN)
    settled = asyncio.run(
        DurableSandboxDeleteCoordinator(service, backend).delete_claimed(
            observed,
            now=NOW,
            retry_at=NOW + timedelta(minutes=1),
        )
    )
    assert settled
    assert backend.probes == []


def test_crash_after_delete_before_settle_retries_idempotent_delete(sessions) -> None:
    service, claimed = _claimed_may_publish(sessions)
    observed = service.observe_create(claimed, now=NOW)
    assert observed is not None
    first_backend = _Backend()
    asyncio.run(
        first_backend.delete_durable_sandbox_strict(observed.backend_lifecycle_digest)
    )
    assert service.list_by_scope(claimed.scope_digest)[0].create_phase == "observed"

    retry_backend = _Backend()
    settled = asyncio.run(
        DurableSandboxDeleteCoordinator(service, retry_backend).delete_claimed(
            observed,
            now=NOW,
            retry_at=NOW + timedelta(minutes=1),
        )
    )
    assert settled
    assert retry_backend.probes == []
    assert retry_backend.deletes == [observed.backend_lifecycle_digest]


def test_terminal_create_deletes_without_probe(sessions) -> None:
    service = DurableSandboxLifecycleService(sessions)
    registered = service.register(_request())
    with sessions() as db:
        _set_task(db, active=True)
        db.commit()
    begun = service.begin_create(registered, now=NOW)
    assert begun is not None
    terminal = service.complete_create(begun, now=NOW, outcome="terminal_absent")
    assert terminal is not None
    with sessions() as db:
        _set_task(db, active=False)
        db.commit()
    claimed = service.claim_for_delete(
        terminal, now=NOW, claim_ttl=timedelta(minutes=1)
    )
    assert claimed is not None
    backend = _Backend(ExactGenerationProbe.UNKNOWN)
    assert asyncio.run(
        DurableSandboxDeleteCoordinator(service, backend).delete_claimed(
            claimed,
            now=NOW,
            retry_at=NOW + timedelta(minutes=1),
        )
    )
    assert backend.probes == []


def test_delete_failure_preserves_observed_tombstone(sessions) -> None:
    service, claimed = _claimed_may_publish(sessions)
    backend = _Backend(ExactGenerationProbe.PRESENT, fail_delete=True)
    settled = asyncio.run(
        DurableSandboxDeleteCoordinator(service, backend).delete_claimed(
            claimed,
            now=NOW,
            retry_at=NOW + timedelta(minutes=1),
        )
    )
    assert not settled
    row = service.list_by_scope(claimed.scope_digest)[0]
    assert row.create_phase == "observed"
    assert row.retry_at.replace(tzinfo=timezone.utc) == NOW + timedelta(minutes=1)