Skip to content

Automate browser based workflows with AI

154 KB
Raw Blame
from __future__ import annotations

import asyncio
from collections.abc import AsyncIterator
from contextlib import asynccontextmanager
from datetime import UTC, datetime, timedelta, tzinfo
from types import SimpleNamespace
from typing import Any
from unittest.mock import AsyncMock, Mock

import pytest
import pytest_asyncio
from sqlalchemy import select, update
from sqlalchemy.ext.asyncio import AsyncEngine

from skyvern.exceptions import BrowserSessionExpired
from skyvern.forge import app
from skyvern.forge.agent import ForgeAgent
from skyvern.forge.sdk.core import skyvern_context
from skyvern.forge.sdk.core.skyvern_context import SkyvernContext
from skyvern.forge.sdk.db.agent_db import AgentDB
from skyvern.forge.sdk.db.enums import BrowserSeedSource
from skyvern.forge.sdk.db.models import (
    BrowserProfileModel,
    CredentialModel,
    PersistentBrowserSessionModel,
    TaskModel,
    TaskRunModel,
    WorkflowModel,
    WorkflowRunAttemptModel,
    WorkflowRunCredentialSelectionModel,
    WorkflowRunModel,
)
from skyvern.forge.sdk.db.repositories import workflow_runs as workflow_runs_repository
from skyvern.forge.sdk.db.repositories.workflow_runs import PrepareNextAttemptResult, WorkflowRunDispatchFinalization
from skyvern.forge.sdk.schemas.files import FileInfo
from skyvern.forge.sdk.schemas.persistent_browser_sessions import EXTERNAL_CDP_BROWSER_VENDOR, BrowserSessionCloseReason
from skyvern.forge.sdk.schemas.tasks import TaskRequest
from skyvern.forge.sdk.workflow import service as service_module
from skyvern.forge.sdk.workflow.browser_profile_key import (
    build_browser_profile_key_digest,
    build_workflow_browser_session_storage_key,
)
from skyvern.forge.sdk.workflow.models.block import BlockType
from skyvern.forge.sdk.workflow.models.workflow import Workflow, WorkflowRequestBody, WorkflowRun, WorkflowRunStatus
from skyvern.forge.sdk.workflow.retry_policy import (
    RETRY_DECISION_FINAL,
    RETRY_DECISION_RETRY,
    RETRY_DECISION_REVOKED,
    RetryDecision,
)
from skyvern.forge.sdk.workflow.service import (
    WorkflowBrowserCleanupResult,
    WorkflowService,
)
from skyvern.schemas.browser_session_kind import BrowserSessionKind
from skyvern.schemas.workflows import BlockStatus, WorkflowRetryPolicy
from skyvern.webeye.persistent_session_errors import BrowserSessionCreditAdmissionRefusal
from skyvern.webeye.real_browser_manager import RealBrowserManager
from tests.unit.force_stub_app import make_workflow_run_attempts_fake
from tests.unit.scoped_asyncio import ScopedAsyncio


def _workflow(browser_profile_key: str | None = None) -> SimpleNamespace:
    return SimpleNamespace(
        persist_browser_session=True,
        reuse_browser_session=False,
        pin_saved_session_ip=False,
        browser_profile_key=browser_profile_key,
        workflow_permanent_id="wpid_test",
        title="Workflow",
    )


def _workflow_run(browser_profile_id: str | None = None) -> SimpleNamespace:
    return SimpleNamespace(
        workflow_run_id="wr_test",
        organization_id="o_test",
        browser_session_id=None,
        browser_profile_id=browser_profile_id,
        start_fresh_browser=None,
        reuse_browser_session=None,
        proxy_location=None,
    )


def _execute_workflow(*, retry_policy: object | None = None) -> SimpleNamespace:
    return SimpleNamespace(
        workflow_id="wf_1",
        persist_browser_session=True,
        reuse_browser_session=False,
        workflow_permanent_id="wpid_test",
        title="Workflow",
        organization_id="o_test",
        generate_script_on_terminal=False,
        model=None,
        workflow_definition=SimpleNamespace(
            parameters=[],
            finally_block_label=None,
            blocks=[SimpleNamespace(block_type=BlockType.TASK)],
            retry_policy=retry_policy,
        ),
    )


def _execute_workflow_run(status: WorkflowRunStatus) -> SimpleNamespace:
    now = datetime.now(UTC)
    return SimpleNamespace(
        workflow_run_id="wr_test",
        workflow_id="wf_1",
        workflow_permanent_id="wpid_test",
        organization_id="o_test",
        created_by=None,
        workflow_schedule_id=None,
        browser_session_id=None,
        browser_profile_id="bp_managed",
        browser_seed_source=BrowserSeedSource.own_memory,
        browser_address=None,
        browser_settings=None,
        start_fresh_browser=None,
        reuse_browser_session=None,
        reuse_bound_key=None,
        status=status,
        failure_reason=None,
        ignore_inherited_workflow_system_prompt=False,
        parent_workflow_run_id=None,
        depends_on_workflow_run_id=None,
        task_queue=None,
        proxy_location=None,
        max_elapsed_time_minutes=1,
        started_at=now,
        created_at=now,
        code_gen=None,
        debug_session_id=None,
        copilot_session_id=None,
        run_with="agent",
    )


def _browser_cleanup_result() -> WorkflowBrowserCleanupResult:
    browser_state = SimpleNamespace(
        browser_artifacts=SimpleNamespace(browser_session_dir="/tmp/fake_profile"),
        browser_context=SimpleNamespace(),
    )
    return WorkflowBrowserCleanupResult(
        browser_state=browser_state,
        tasks=[],
        all_workflow_task_ids=[],
        child_workflow_run_ids=[],
        close_browser_on_completion=True,
    )


def _patch_execute_workflow_deps(
    monkeypatch: pytest.MonkeyPatch,
    svc: WorkflowService,
    workflow: SimpleNamespace,
    refreshed_run: SimpleNamespace,
) -> None:
    created_run = _execute_workflow_run(WorkflowRunStatus.created)
    running_run = _execute_workflow_run(WorkflowRunStatus.running)
    workflow_run_context = SimpleNamespace(
        browser_session_id=None,
        drain_failure_evidence_capture=AsyncMock(),
    )
    workflow_context_manager = SimpleNamespace(
        initialize_workflow_run_context=AsyncMock(),
        has_workflow_run_context=lambda _workflow_run_id: True,
        get_workflow_run_context=lambda _workflow_run_id: workflow_run_context,
        remove_workflow_run_context=lambda _workflow_run_id: None,
    )
    database = SimpleNamespace(
        workflow_runs=SimpleNamespace(
            get_workflow_run=AsyncMock(return_value=refreshed_run),
            update_workflow_run=AsyncMock(),
            # Conditional finalize used by mark_workflow_run_as_failed_if_not_final: returns the
            # run when this caller won the terminal transition, None when it was already final.
            update_workflow_run_if_not_final=AsyncMock(return_value=refreshed_run),
            get_workflow_runs_by_parent_workflow_run_id=AsyncMock(return_value=[]),
        ),
        debug=SimpleNamespace(has_block_run_for_workflow_run=AsyncMock(return_value=False)),
        artifacts=SimpleNamespace(claim_session_download_artifacts_for_run=AsyncMock(return_value=0)),
        workflow_run_attempts=make_workflow_run_attempts_fake(),
    )

    monkeypatch.setattr(service_module.app, "WORKFLOW_CONTEXT_MANAGER", workflow_context_manager)
    monkeypatch.setattr(service_module.app, "DATABASE", database)
    monkeypatch.setattr(service_module.app.ARTIFACT_MANAGER, "wait_for_upload_aiotasks", AsyncMock())
    monkeypatch.setattr(service_module.app.STORAGE, "save_downloaded_files", AsyncMock())
    monkeypatch.setattr(service_module.workflow_script_service, "workflow_has_conditionals", lambda _workflow: False)
    monkeypatch.setattr(
        service_module.workflow_script_service,
        "get_workflow_script",
        AsyncMock(return_value=(None, None, False)),
    )
    monkeypatch.setattr(service_module.skyvern_context, "current", lambda: None)
    monkeypatch.setattr(service_module, "is_adaptive_caching", lambda _workflow, _workflow_run: False)

    monkeypatch.setattr(app.AGENT_FUNCTION, "before_workflow_run_start", AsyncMock())
    monkeypatch.setattr(app.AGENT_FUNCTION, "should_defer_workflow_browser_creation", AsyncMock(return_value=False))
    monkeypatch.setattr(svc, "get_workflow_run", AsyncMock(return_value=created_run))
    monkeypatch.setattr(svc, "get_workflow_by_workflow_run_id", AsyncMock(return_value=workflow))
    monkeypatch.setattr(svc, "bind_browser_action_policy", AsyncMock(return_value=None))
    monkeypatch.setattr(svc, "mark_workflow_run_as_running", AsyncMock(return_value=running_run))
    monkeypatch.setattr(service_module, "mark_attempt_started", AsyncMock(return_value=True))
    monkeypatch.setattr(svc, "get_workflow_run_parameter_tuples", AsyncMock(return_value=[]))
    monkeypatch.setattr(svc, "get_workflow_output_parameters", AsyncMock(return_value=[]))
    monkeypatch.setattr(svc, "_collect_inherited_workflow_system_prompt", AsyncMock(return_value=None))
    monkeypatch.setattr(svc, "auto_create_browser_session_if_needed", AsyncMock(return_value=None))
    monkeypatch.setattr(svc, "_browser_profile_is_managed", AsyncMock(return_value=False))
    monkeypatch.setattr(svc, "_execute_workflow_blocks", AsyncMock(return_value=(refreshed_run, set())))
    monkeypatch.setattr(svc, "generate_script_if_needed", AsyncMock())
    monkeypatch.setattr(svc, "should_run_script", AsyncMock(return_value=False))


def _patch_browser_cleanup(monkeypatch: pytest.MonkeyPatch, svc: WorkflowService, order: list[str]) -> AsyncMock:
    clean_up_browser = AsyncMock(side_effect=lambda **_kwargs: order.append("teardown") or _browser_cleanup_result())
    monkeypatch.setattr(svc, "_clean_up_workflow_browser", clean_up_browser)
    return clean_up_browser


def _patch_finalize(
    monkeypatch: pytest.MonkeyPatch,
    svc: WorkflowService,
    order: list[str],
    finalized_run: SimpleNamespace,
) -> None:
    monkeypatch.setattr(
        svc,
        "_finalize_workflow_run_status",
        AsyncMock(side_effect=lambda **_kwargs: order.append("finalize") or finalized_run),
    )


async def _run_execute_workflow(svc: WorkflowService, **kwargs: Any) -> SimpleNamespace:
    return await svc.execute_workflow(
        workflow_run_id="wr_test",
        api_key=None,
        organization=SimpleNamespace(organization_id="o_test"),
        **kwargs,
    )


def _mock_storage(monkeypatch: pytest.MonkeyPatch, *, legacy_dir: str | None) -> AsyncMock:
    monkeypatch.setattr(app.STORAGE, "retrieve_browser_session", AsyncMock(return_value=legacy_dir))
    store = AsyncMock()
    monkeypatch.setattr(app.STORAGE, "store_browser_profile", store)
    return store


@pytest.mark.asyncio
async def test_ensure_managed_browser_profile_returns_managed_id(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    profile = SimpleNamespace(browser_profile_id="bp_managed")
    get_or_create = AsyncMock(return_value=(profile, True))
    monkeypatch.setattr(app.DATABASE.browser_sessions, "get_or_create_managed_browser_profile", get_or_create)
    _mock_storage(monkeypatch, legacy_dir=None)

    result = await WorkflowService()._ensure_managed_browser_profile(
        workflow=_workflow(browser_profile_key="{{ credential_id }}"),
        workflow_run=_workflow_run(),
        parameter_values={"credential_id": "cred_123"},
    )

    assert result == "bp_managed"
    get_or_create.assert_awaited_once_with(
        organization_id="o_test",
        workflow_permanent_id="wpid_test",
        browser_profile_key_digest=build_browser_profile_key_digest("cred_123"),
        name="Workflow (auto-saved: cred_123)",
    )


@pytest.mark.asyncio
async def test_ensure_managed_browser_profile_uses_empty_digest_for_unkeyed_workflow(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    profile = SimpleNamespace(browser_profile_id="bp_managed")
    get_or_create = AsyncMock(return_value=(profile, True))
    monkeypatch.setattr(app.DATABASE.browser_sessions, "get_or_create_managed_browser_profile", get_or_create)
    _mock_storage(monkeypatch, legacy_dir=None)

    await WorkflowService()._ensure_managed_browser_profile(
        workflow=_workflow(),
        workflow_run=_workflow_run(),
        parameter_values={},
    )

    get_or_create.assert_awaited_once_with(
        organization_id="o_test",
        workflow_permanent_id="wpid_test",
        browser_profile_key_digest="",
        name="Workflow (auto-saved session)",
    )


@pytest.mark.asyncio
async def test_ensure_managed_browser_profile_skips_non_persist_workflow(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    get_or_create = AsyncMock()
    monkeypatch.setattr(app.DATABASE.browser_sessions, "get_or_create_managed_browser_profile", get_or_create)
    workflow = _workflow()
    workflow.persist_browser_session = False

    result = await WorkflowService()._ensure_managed_browser_profile(
        workflow=workflow,
        workflow_run=_workflow_run(),
        parameter_values={},
    )

    assert result is None
    get_or_create.assert_not_awaited()


@pytest.mark.asyncio
async def test_ensure_managed_browser_profile_seeds_new_profile_from_legacy_archive(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    profile = SimpleNamespace(browser_profile_id="bp_managed")
    get_or_create = AsyncMock(return_value=(profile, True))
    monkeypatch.setattr(app.DATABASE.browser_sessions, "get_or_create_managed_browser_profile", get_or_create)
    store = _mock_storage(monkeypatch, legacy_dir="/tmp/legacy_session")

    await WorkflowService()._ensure_managed_browser_profile(
        workflow=_workflow(),
        workflow_run=_workflow_run(),
        parameter_values={},
    )

    store.assert_awaited_once_with(
        "o_test",
        profile_id="bp_managed",
        directory="/tmp/legacy_session",
    )


@pytest.mark.asyncio
async def test_ensure_managed_browser_profile_rolls_back_on_seed_failure(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    profile = SimpleNamespace(browser_profile_id="bp_managed")
    monkeypatch.setattr(
        app.DATABASE.browser_sessions,
        "get_or_create_managed_browser_profile",
        AsyncMock(return_value=(profile, True)),
    )
    hard_delete = AsyncMock()
    monkeypatch.setattr(app.DATABASE.browser_sessions, "hard_delete_browser_profile", hard_delete)
    monkeypatch.setattr(app.STORAGE, "retrieve_browser_session", AsyncMock(return_value="/tmp/legacy_session"))
    monkeypatch.setattr(app.STORAGE, "store_browser_profile", AsyncMock(side_effect=RuntimeError("upload failed")))

    result = await WorkflowService()._ensure_managed_browser_profile(
        workflow=_workflow(),
        workflow_run=_workflow_run(),
        parameter_values={},
    )

    assert result is None
    hard_delete.assert_awaited_once_with(profile_id="bp_managed", organization_id="o_test")


@pytest.mark.asyncio
async def test_ensure_managed_browser_profile_does_not_seed_existing_profile(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    profile = SimpleNamespace(browser_profile_id="bp_managed")
    get_or_create = AsyncMock(return_value=(profile, False))
    monkeypatch.setattr(app.DATABASE.browser_sessions, "get_or_create_managed_browser_profile", get_or_create)
    retrieve = AsyncMock(return_value="/tmp/legacy_session")
    monkeypatch.setattr(app.STORAGE, "retrieve_browser_session", retrieve)
    store = AsyncMock()
    monkeypatch.setattr(app.STORAGE, "store_browser_profile", store)

    result = await WorkflowService()._ensure_managed_browser_profile(
        workflow=_workflow(),
        workflow_run=_workflow_run(),
        parameter_values={},
    )

    assert result == "bp_managed"
    retrieve.assert_not_awaited()
    store.assert_not_awaited()


def test_legacy_storage_key_matches_managed_digest_for_keyed_workflow() -> None:
    # Read-compat invariant: the managed profile digest and the legacy archive segment must
    # derive from the same rendered key, so seeding finds the right archive.
    rendered = "cred_123"
    storage_key = build_workflow_browser_session_storage_key("wpid_test", rendered)
    assert storage_key.endswith(build_browser_profile_key_digest(rendered))


@pytest.mark.asyncio
async def test_auto_create_browser_session_for_human_interaction_loads_managed_profile(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    create_session = AsyncMock(return_value=SimpleNamespace(persistent_browser_session_id="pbs_test"))
    monkeypatch.setattr(app.PERSISTENT_SESSIONS_MANAGER, "create_session", create_session)
    workflow = SimpleNamespace(
        workflow_definition=SimpleNamespace(
            blocks=[SimpleNamespace(block_type=BlockType.HUMAN_INTERACTION, timeout_seconds=60)]
        )
    )

    browser_session = await WorkflowService().auto_create_browser_session_if_needed(
        "o_test",
        workflow,
        browser_profile_id="bp_managed",
    )

    assert browser_session.persistent_browser_session_id == "pbs_test"
    create_session.assert_awaited_once_with(
        organization_id="o_test",
        timeout_minutes=61,
        browser_profile_id="bp_managed",
        proxy_location=None,
        inherit_profile_proxy=True,
        session_kind=BrowserSessionKind.workflow_run,
        browser_settings=None,
        created_for_workflow_run_id=None,
    )


@pytest.mark.asyncio
async def test_browser_profile_is_managed_distinguishes_user_profiles(monkeypatch: pytest.MonkeyPatch) -> None:
    get_browser_profile = AsyncMock(return_value=SimpleNamespace(is_managed=False))
    monkeypatch.setattr(app.DATABASE.browser_sessions, "get_browser_profile", get_browser_profile)

    is_managed = await WorkflowService()._browser_profile_is_managed(
        organization_id="o_test",
        browser_profile_id="bp_user",
    )

    assert is_managed is False
    get_browser_profile.assert_awaited_once_with(profile_id="bp_user", organization_id="o_test")


@pytest.mark.asyncio
async def test_browser_profile_is_managed_detects_managed_profile(monkeypatch: pytest.MonkeyPatch) -> None:
    get_browser_profile = AsyncMock(return_value=SimpleNamespace(is_managed=True))
    monkeypatch.setattr(app.DATABASE.browser_sessions, "get_browser_profile", get_browser_profile)

    is_managed = await WorkflowService()._browser_profile_is_managed(
        organization_id="o_test",
        browser_profile_id="bp_managed",
    )

    assert is_managed is True


@pytest.mark.asyncio
async def test_execute_workflow_persists_managed_profile_before_final_status(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    workflow = _execute_workflow()
    completed_run = _execute_workflow_run(WorkflowRunStatus.completed)
    order: list[str] = []

    svc = WorkflowService()
    _patch_execute_workflow_deps(monkeypatch, svc, workflow, _execute_workflow_run(WorkflowRunStatus.running))
    clean_up_browser = _patch_browser_cleanup(monkeypatch, svc, order)
    monkeypatch.setattr(
        svc,
        "_persist_workflow_browser_session_if_needed",
        AsyncMock(side_effect=lambda **_kwargs: order.append("store")),
    )
    _patch_finalize(monkeypatch, svc, order, completed_run)
    monkeypatch.setattr(svc, "persist_video_data", AsyncMock(side_effect=lambda *a, **k: order.append("video")))
    monkeypatch.setattr(svc, "execute_workflow_webhook", AsyncMock(side_effect=lambda *a, **k: order.append("webhook")))

    result = await _run_execute_workflow(svc)

    assert result is completed_run
    assert order == ["teardown", "store", "finalize", "video", "webhook"]
    clean_up_browser.assert_awaited_once()
    svc._persist_workflow_browser_session_if_needed.assert_awaited_once()


@pytest.mark.asyncio
async def test_execute_workflow_with_attempt_row_still_records_final_webhook(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    workflow = _execute_workflow(retry_policy=WorkflowRetryPolicy(max_retries=1, retry_on=[{"status": "failed"}]))
    completed_run = _execute_workflow_run(WorkflowRunStatus.completed)
    order: list[str] = []

    svc = WorkflowService()
    _patch_execute_workflow_deps(monkeypatch, svc, workflow, _execute_workflow_run(WorkflowRunStatus.running))
    _patch_browser_cleanup(monkeypatch, svc, order)
    monkeypatch.setattr(
        svc,
        "_persist_workflow_browser_session_if_needed",
        AsyncMock(side_effect=lambda **_kwargs: order.append("store")),
    )
    _patch_finalize(monkeypatch, svc, order, completed_run)
    monkeypatch.setattr(svc, "persist_video_data", AsyncMock(side_effect=lambda *a, **k: order.append("video")))
    webhook = AsyncMock(side_effect=lambda *a, **k: order.append("webhook"))
    monkeypatch.setattr(svc, "execute_workflow_webhook", webhook)

    attempt = SimpleNamespace(attempt_number=1, retry_decision=RETRY_DECISION_FINAL)
    service_module.app.DATABASE.workflow_run_attempts.get_attempts = AsyncMock(return_value=[attempt])
    monkeypatch.setattr(
        service_module.app.DATABASE.workflow_run_attempts,
        "claim_attempt_side_effects",
        AsyncMock(return_value=True),
    )
    monkeypatch.setattr(
        service_module,
        "get_recorded_decision",
        AsyncMock(return_value=RetryDecision(False, 1, 0, True, RETRY_DECISION_FINAL)),
    )
    result = await _run_execute_workflow(svc)

    assert result is completed_run
    assert order == ["teardown", "store", "finalize", "video", "webhook"]
    webhook.assert_awaited_once_with(
        completed_run,
        None,
        claim_kind="final",
        skip_side_effects_lease_check=True,
    )


@pytest.mark.asyncio
@pytest.mark.parametrize("claim_replaced", [False, True], ids=["claim_current", "claim_replaced"])
async def test_execute_workflow_with_retries_stops_when_its_dispatch_claim_was_replaced(
    monkeypatch: pytest.MonkeyPatch, claim_replaced: bool
) -> None:
    """The sweep releases a stale dispatch claim and re-issues it to another dispatch. The superseded
    dispatch must neither run blocks nor fail the run its new owner is executing."""
    workflow = _execute_workflow(retry_policy=WorkflowRetryPolicy(max_retries=1, retry_on=[{"status": "failed"}]))
    final_run = _execute_workflow_run(WorkflowRunStatus.completed)
    order: list[str] = []

    svc = WorkflowService()
    _patch_execute_workflow_deps(monkeypatch, svc, workflow, final_run)
    _patch_browser_cleanup(monkeypatch, svc, order)
    monkeypatch.setattr(svc, "_persist_workflow_browser_session_if_needed", AsyncMock())
    _patch_finalize(monkeypatch, svc, order, final_run)
    monkeypatch.setattr(svc, "persist_video_data", AsyncMock())
    monkeypatch.setattr(svc, "execute_workflow_webhook", AsyncMock())
    monkeypatch.setattr(svc, "_run_terminal_side_effects_with_retries", AsyncMock())
    monkeypatch.setattr(service_module, "mark_attempt_started", AsyncMock(return_value=False))
    claimed_at = datetime.now(UTC).replace(tzinfo=None)
    row_started_at = claimed_at + timedelta(seconds=1) if claim_replaced else claimed_at
    attempt = SimpleNamespace(
        attempt_number=1, retry_decision=None, started_at=row_started_at, pinned_browser_session_id=None
    )
    service_module.app.DATABASE.workflow_run_attempts.get_attempts = AsyncMock(return_value=[attempt])
    monkeypatch.setattr(
        service_module.app.DATABASE.workflow_run_attempts,
        "claim_attempt_side_effects",
        AsyncMock(return_value=True),
    )
    monkeypatch.setattr(
        service_module,
        "get_recorded_decision",
        AsyncMock(return_value=RetryDecision(False, 1, 0, True, RETRY_DECISION_FINAL)),
    )
    fail_run = AsyncMock()
    monkeypatch.setattr(svc, "mark_workflow_run_as_failed_if_not_final", fail_run)

    result = await svc.execute_workflow_with_retries(
        workflow_run_id="wr_test",
        api_key=None,
        organization=SimpleNamespace(organization_id="o_test"),
        prepared_attempt_claimed=True,
        dispatch_claim_started_at=claimed_at,
    )

    if claim_replaced:
        # The superseded owner hands back the run as currently read, untouched.
        assert result is svc.get_workflow_run.return_value
        assert svc._execute_workflow_blocks.await_count == 0
    else:
        assert result is final_run
        assert svc._execute_workflow_blocks.await_count == 1
    fail_run.assert_not_awaited()


@pytest.mark.asyncio
@pytest.mark.parametrize(
    ("attempt", "expected_block_runs"),
    [
        (SimpleNamespace(attempt_number=1, retry_decision=RETRY_DECISION_REVOKED, started_at=None), 0),
        (SimpleNamespace(attempt_number=1, retry_decision=None, started_at=datetime.now(UTC)), 1),
    ],
    ids=["cancel_finalized_the_attempt", "activity_re_entered_a_started_attempt"],
)
async def test_execute_workflow_stops_when_the_attempt_was_finalized_before_it_started(
    monkeypatch: pytest.MonkeyPatch,
    attempt: SimpleNamespace,
    expected_block_runs: int,
) -> None:
    """The attempt-start CAS loses to a cancel that lands after the running write; the run must not
    resolve secrets or run blocks after its final webhook. A re-entered activity still continues."""
    workflow = _execute_workflow(retry_policy=WorkflowRetryPolicy(max_retries=1, retry_on=[{"status": "failed"}]))
    final_status = WorkflowRunStatus.canceled if expected_block_runs == 0 else WorkflowRunStatus.completed
    final_run = _execute_workflow_run(final_status)
    order: list[str] = []

    svc = WorkflowService()
    _patch_execute_workflow_deps(monkeypatch, svc, workflow, final_run)
    _patch_browser_cleanup(monkeypatch, svc, order)
    monkeypatch.setattr(svc, "_persist_workflow_browser_session_if_needed", AsyncMock())
    _patch_finalize(monkeypatch, svc, order, final_run)
    monkeypatch.setattr(svc, "persist_video_data", AsyncMock())
    monkeypatch.setattr(svc, "execute_workflow_webhook", AsyncMock())
    monkeypatch.setattr(service_module, "mark_attempt_started", AsyncMock(return_value=False))
    service_module.app.DATABASE.workflow_run_attempts.get_attempts = AsyncMock(return_value=[attempt])
    monkeypatch.setattr(
        service_module.app.DATABASE.workflow_run_attempts,
        "claim_attempt_side_effects",
        AsyncMock(return_value=True),
    )
    monkeypatch.setattr(
        service_module,
        "get_recorded_decision",
        AsyncMock(return_value=RetryDecision(False, 1, 0, True, RETRY_DECISION_FINAL)),
    )

    result = await _run_execute_workflow(svc)

    assert result is final_run
    assert svc._execute_workflow_blocks.await_count == expected_block_runs
    assert svc.get_workflow_run_parameter_tuples.await_count == expected_block_runs


@pytest.mark.asyncio
async def test_execute_workflow_abandons_retry_for_non_retry_aware_caller(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    workflow = _execute_workflow(retry_policy=WorkflowRetryPolicy(max_retries=1, retry_on=[{"status": "failed"}]))
    completed_run = _execute_workflow_run(WorkflowRunStatus.completed)
    completed_run.finished_at = datetime.now(UTC)
    order: list[str] = []

    svc = WorkflowService()
    _patch_execute_workflow_deps(monkeypatch, svc, workflow, _execute_workflow_run(WorkflowRunStatus.running))
    _patch_browser_cleanup(monkeypatch, svc, order)
    monkeypatch.setattr(
        svc,
        "_persist_workflow_browser_session_if_needed",
        AsyncMock(side_effect=lambda **_kwargs: order.append("store")),
    )
    _patch_finalize(monkeypatch, svc, order, completed_run)
    monkeypatch.setattr(svc, "persist_video_data", AsyncMock(side_effect=lambda *a, **k: order.append("video")))
    webhook = AsyncMock(side_effect=lambda *a, **k: order.append("webhook"))
    monkeypatch.setattr(svc, "execute_workflow_webhook", webhook)

    attempt = SimpleNamespace(attempt_number=1, retry_decision=RETRY_DECISION_RETRY)
    service_module.app.DATABASE.workflow_run_attempts.get_attempts = AsyncMock(return_value=[attempt])
    monkeypatch.setattr(
        service_module.app.DATABASE.workflow_run_attempts,
        "claim_attempt_side_effects",
        AsyncMock(return_value=True),
    )
    monkeypatch.setattr(
        service_module,
        "get_recorded_decision",
        AsyncMock(return_value=RetryDecision(True, 1, 5, False, None)),
    )
    finalize_abandoned = AsyncMock(return_value=RetryDecision(False, 1, 0, True, "caller_not_retry_aware"))
    monkeypatch.setattr(service_module, "finalize_abandoned_attempt", finalize_abandoned)

    result = await _run_execute_workflow(svc)

    assert result is completed_run
    assert order == ["teardown", "store", "finalize", "video", "webhook"]
    finalize_abandoned.assert_awaited_once_with(
        workflow_run_id="wr_test",
        organization_id="o_test",
        reason="caller_not_retry_aware",
        status=WorkflowRunStatus.completed,
        failure_reason=None,
        finished_at=completed_run.finished_at,
        attempt_number=1,
    )
    webhook.assert_awaited_once_with(
        completed_run,
        None,
        claim_kind="final",
        skip_side_effects_lease_check=True,
    )


@pytest.mark.asyncio
async def test_execute_workflow_with_retries_retries_escaped_exception(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    svc = WorkflowService()
    organization = SimpleNamespace(organization_id="o_test")
    failed_run = _execute_workflow_run(WorkflowRunStatus.failed)
    completed_run = _execute_workflow_run(WorkflowRunStatus.completed)
    attempt_rows = [
        SimpleNamespace(
            attempt_number=1,
            retry_decision=None,
            finished_at=None,
            decision_reason=None,
            next_attempt_at=None,
        )
    ]
    monkeypatch.setattr(
        service_module.app,
        "DATABASE",
        SimpleNamespace(
            workflow_run_attempts=SimpleNamespace(
                get_attempts=AsyncMock(return_value=attempt_rows),
                claim_prepared_attempt_execution=AsyncMock(return_value=datetime(2026, 1, 2, tzinfo=UTC)),
            ),
            workflow_runs=SimpleNamespace(
                get_workflow_run=AsyncMock(return_value=completed_run),
            ),
        ),
    )

    async def execute_attempt(**kwargs: Any) -> WorkflowRun:
        kwargs["on_execution_authority"](kwargs["attempt_number"], kwargs["dispatch_claim_started_at"])
        if kwargs["attempt_number"] == 1:
            raise RuntimeError("escaped")
        return completed_run

    execute = AsyncMock(side_effect=execute_attempt)
    monkeypatch.setattr(svc, "execute_workflow", execute)
    mark_failed = AsyncMock(
        return_value=WorkflowRunDispatchFinalization("accepted", failed_run, 1, None, False, transitioned=True)
    )
    monkeypatch.setattr(svc, "finalize_workflow_run_for_dispatch", mark_failed)
    side_effects = AsyncMock()
    monkeypatch.setattr(svc, "run_terminal_side_effects", side_effects)
    decisions = [
        RetryDecision(True, 1, 0, False, "matched"),
        RetryDecision(True, 1, 0, False, "matched"),
        RetryDecision(False, 2, 0, True, "no_match"),
    ]
    monkeypatch.setattr(service_module, "get_recorded_decision", AsyncMock(side_effect=decisions))
    prepare = AsyncMock(
        return_value=PrepareNextAttemptResult(
            status="inserted",
            pinned_browser_session_id="browser-2",
        )
    )
    monkeypatch.setattr(service_module, "prepare_next_attempt_result", prepare)

    result = await svc.execute_workflow_with_retries(
        workflow_run_id="wr_test",
        api_key=None,
        organization=organization,
    )

    assert result is completed_run
    assert [call.kwargs["attempt_number"] for call in execute.await_args_list] == [1, 2]
    assert execute.await_args_list[1].kwargs["dispatch_claim_started_at"] == datetime(2026, 1, 2, tzinfo=UTC)
    mark_failed.assert_awaited_once_with(
        workflow_run_id="wr_test",
        failure_reason="escaped",
        organization_id="o_test",
        attempt_number=1,
        dispatch_claim_started_at=None,
        status=WorkflowRunStatus.failed,
        pre_execution=False,
    )
    prepare.assert_awaited_once_with(
        workflow_run_id="wr_test",
        organization_id="o_test",
        from_attempt=1,
        clear_browser_address=False,
    )
    assert [call.args[1] for call in side_effects.await_args_list] == [decisions[0], decisions[2]]


@pytest.mark.asyncio
async def test_execute_workflow_with_retries_leaves_pending_retry_for_recovery_on_cancellation(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    svc = WorkflowService()
    organization = SimpleNamespace(organization_id="o_test")
    completed_run = _execute_workflow_run(WorkflowRunStatus.completed)
    completed_run.finished_at = datetime.now(UTC)
    attempt_rows = [
        SimpleNamespace(
            attempt_number=1,
            retry_decision=None,
            finished_at=None,
            decision_reason=None,
            next_attempt_at=None,
        )
    ]
    monkeypatch.setattr(
        service_module.app,
        "DATABASE",
        SimpleNamespace(
            workflow_run_attempts=SimpleNamespace(get_attempts=AsyncMock(return_value=attempt_rows)),
            workflow_runs=SimpleNamespace(
                get_workflow_run=AsyncMock(return_value=completed_run),
            ),
        ),
    )
    monkeypatch.setattr(svc, "execute_workflow", AsyncMock(return_value=completed_run))
    retry_decision = RetryDecision(True, 1, 30, False, "matched")
    monkeypatch.setattr(service_module, "get_recorded_decision", AsyncMock(return_value=retry_decision))

    side_effects_started = asyncio.Event()

    async def record_side_effects(*_args: Any, **_kwargs: Any) -> None:
        side_effects_started.set()

    side_effects = AsyncMock(side_effect=record_side_effects)
    monkeypatch.setattr(svc, "run_terminal_side_effects", side_effects)
    prepare = AsyncMock()
    monkeypatch.setattr(service_module, "prepare_next_attempt_result", prepare)
    finalize_abandoned = AsyncMock()
    monkeypatch.setattr(service_module, "finalize_abandoned_attempt", finalize_abandoned)

    task = asyncio.create_task(
        svc.execute_workflow_with_retries(
            workflow_run_id="wr_test",
            api_key=None,
            organization=organization,
        )
    )
    await asyncio.wait_for(side_effects_started.wait(), timeout=1)
    task.cancel()

    with pytest.raises(asyncio.CancelledError):
        await task

    # Shutdown must not convert the durable retry; recovery resumes it on the next start.
    finalize_abandoned.assert_not_awaited()
    assert [call.args[1] for call in side_effects.await_args_list] == [retry_decision]
    prepare.assert_not_awaited()


@pytest.mark.asyncio
async def test_execute_workflow_with_retries_sleeps_until_the_recorded_next_attempt(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    # The decision stamps next_attempt_at before cleanup and interim webhooks run; the owner must not
    # add the full policy delay on top of that work.
    svc = WorkflowService()
    organization = SimpleNamespace(organization_id="o_test")
    completed_run = _execute_workflow_run(WorkflowRunStatus.completed)
    completed_run.finished_at = datetime.now(UTC)
    attempt_rows = [
        SimpleNamespace(
            attempt_number=1, retry_decision=None, finished_at=None, decision_reason=None, next_attempt_at=None
        )
    ]
    monkeypatch.setattr(
        service_module.app,
        "DATABASE",
        SimpleNamespace(
            workflow_run_attempts=SimpleNamespace(get_attempts=AsyncMock(return_value=attempt_rows)),
            workflow_runs=SimpleNamespace(get_workflow_run=AsyncMock(return_value=completed_run)),
        ),
    )
    monkeypatch.setattr(svc, "execute_workflow", AsyncMock(return_value=completed_run))
    retry_decision = RetryDecision(
        True, 1, 30, False, "matched", next_attempt_at=datetime.now(UTC) + timedelta(seconds=2)
    )
    monkeypatch.setattr(service_module, "get_recorded_decision", AsyncMock(return_value=retry_decision))
    monkeypatch.setattr(svc, "run_terminal_side_effects", AsyncMock())
    monkeypatch.setattr(svc, "_prepare_recorded_retry", AsyncMock(return_value=None))
    sleep = AsyncMock()
    monkeypatch.setattr(service_module, "asyncio", ScopedAsyncio(sleep=sleep))

    result = await svc.execute_workflow_with_retries(workflow_run_id="wr_test", api_key=None, organization=organization)

    assert result is completed_run
    (slept,) = (call.args[0] for call in sleep.await_args_list)
    assert 0 < slept <= 2


@pytest.mark.asyncio
async def test_execute_workflow_with_retries_re_raises_escaped_exception_after_final_effects(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    svc = WorkflowService()
    organization = SimpleNamespace(organization_id="o_test")
    running_run = _execute_workflow_run(WorkflowRunStatus.running)
    failed_run = _execute_workflow_run(WorkflowRunStatus.failed)
    attempt_rows = [SimpleNamespace(attempt_number=1)]
    get_run = AsyncMock(return_value=running_run)
    monkeypatch.setattr(
        service_module.app,
        "DATABASE",
        SimpleNamespace(
            workflow_run_attempts=SimpleNamespace(get_attempts=AsyncMock(return_value=attempt_rows)),
            workflow_runs=SimpleNamespace(get_workflow_run=get_run),
        ),
    )
    error = RuntimeError("escaped")

    async def execute_attempt(**kwargs: Any) -> WorkflowRun:
        kwargs["on_execution_authority"](kwargs["attempt_number"], kwargs["dispatch_claim_started_at"])
        raise error

    monkeypatch.setattr(svc, "execute_workflow", AsyncMock(side_effect=execute_attempt))
    mark_failed = AsyncMock(
        return_value=WorkflowRunDispatchFinalization("accepted", failed_run, 1, None, False, transitioned=True)
    )
    monkeypatch.setattr(svc, "finalize_workflow_run_for_dispatch", mark_failed)
    side_effects = AsyncMock()
    monkeypatch.setattr(svc, "run_terminal_side_effects", side_effects)
    final_decision = RetryDecision(False, 1, 0, True, "no_match")
    monkeypatch.setattr(service_module, "get_recorded_decision", AsyncMock(return_value=final_decision))

    with pytest.raises(RuntimeError, match="escaped") as raised:
        await svc.execute_workflow_with_retries(
            workflow_run_id="wr_test",
            api_key=None,
            organization=organization,
        )

    assert raised.value is error
    get_run.assert_not_awaited()
    mark_failed.assert_awaited_once_with(
        workflow_run_id="wr_test",
        failure_reason="escaped",
        organization_id="o_test",
        attempt_number=1,
        dispatch_claim_started_at=None,
        status=WorkflowRunStatus.failed,
        pre_execution=False,
    )
    side_effects.assert_awaited_once()
    assert side_effects.await_args.args == (failed_run, final_decision)
    assert side_effects.await_args.kwargs["api_key"] is None


@pytest.mark.asyncio
async def test_cleanup_claims_session_downloads_from_retry_attempt_start(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    svc = WorkflowService()
    workflow = _execute_workflow(retry_policy=WorkflowRetryPolicy(max_retries=1, retry_on=[{"status": "failed"}]))
    workflow_run = _execute_workflow_run(WorkflowRunStatus.completed)
    workflow_run.created_at = datetime(2026, 9, 7, 10, 0, tzinfo=UTC)
    attempt_started_at = datetime(2026, 9, 7, 12, 0, tzinfo=UTC)
    attempt = SimpleNamespace(attempt_number=2, started_at=attempt_started_at)
    claim = AsyncMock(return_value=1)
    monkeypatch.setattr(
        service_module.app,
        "DATABASE",
        SimpleNamespace(
            workflow_run_attempts=SimpleNamespace(get_attempts=AsyncMock(return_value=[attempt])),
            debug=SimpleNamespace(has_block_run_for_workflow_run=AsyncMock(return_value=False)),
            artifacts=SimpleNamespace(claim_session_download_artifacts_for_run=claim),
        ),
    )
    monkeypatch.setattr(
        service_module.app,
        "AGENT_FUNCTION",
        SimpleNamespace(
            on_workflow_run_terminal=AsyncMock(),
            is_block_scoped_workflow_run=AsyncMock(return_value=False),
        ),
    )
    monkeypatch.setattr(
        service_module.app,
        "WORKFLOW_CONTEXT_MANAGER",
        SimpleNamespace(
            remove_workflow_run_context=lambda _workflow_run_id: None,
            has_workflow_run_context=lambda _workflow_run_id: False,
        ),
    )
    monkeypatch.setattr(
        service_module.skyvern_context,
        "current",
        lambda: SkyvernContext(browser_session_id="bs_retry", run_id=workflow_run.workflow_run_id),
    )
    monkeypatch.setattr(service_module.app.STORAGE, "save_downloaded_files", AsyncMock())
    monkeypatch.setattr(service_module.app.ARTIFACT_MANAGER, "wait_for_upload_aiotasks", AsyncMock())
    monkeypatch.setattr(
        service_module, "get_recorded_decision", AsyncMock(return_value=RetryDecision(False, 2, 0, True, "final"))
    )
    monkeypatch.setattr(svc, "run_terminal_side_effects", AsyncMock())

    await svc.clean_up_workflow(
        workflow=workflow,
        workflow_run=workflow_run,
        browser_cleanup_result=WorkflowBrowserCleanupResult(
            browser_state=None,
            tasks=[],
            all_workflow_task_ids=[],
            child_workflow_run_ids=[],
            close_browser_on_completion=True,
        ),
        attempt_number=2,
    )

    claim.assert_awaited_once_with(
        run_id=workflow_run.workflow_run_id,
        browser_session_id="bs_retry",
        organization_id=workflow_run.organization_id,
        run_started_at=attempt_started_at,
    )


@pytest.mark.asyncio
async def test_cleanup_uses_legacy_path_when_retry_attempt_lookup_fails(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    svc = WorkflowService()
    workflow = _execute_workflow()
    workflow_run = _execute_workflow_run(WorkflowRunStatus.completed)
    get_attempts = AsyncMock(side_effect=RuntimeError("attempt lookup unavailable"))
    monkeypatch.setattr(
        service_module.app,
        "DATABASE",
        SimpleNamespace(
            workflow_run_attempts=SimpleNamespace(get_attempts=get_attempts),
            debug=SimpleNamespace(has_block_run_for_workflow_run=AsyncMock(return_value=False)),
        ),
    )
    terminal_hook = AsyncMock()
    monkeypatch.setattr(
        service_module.app,
        "AGENT_FUNCTION",
        SimpleNamespace(
            on_workflow_run_terminal=terminal_hook,
            is_block_scoped_workflow_run=AsyncMock(return_value=False),
        ),
    )
    monkeypatch.setattr(
        service_module.app,
        "WORKFLOW_CONTEXT_MANAGER",
        SimpleNamespace(remove_workflow_run_context=Mock(), has_workflow_run_context=lambda _workflow_run_id: False),
    )
    monkeypatch.setattr(service_module.skyvern_context, "current", lambda: None)
    save_downloads = AsyncMock()
    monkeypatch.setattr(service_module.app.STORAGE, "save_downloaded_files", save_downloads)
    wait_for_uploads = AsyncMock()
    monkeypatch.setattr(service_module.app.ARTIFACT_MANAGER, "wait_for_upload_aiotasks", wait_for_uploads)
    delete_attachments = AsyncMock()
    monkeypatch.setattr(service_module.uploaded_file_service, "delete_files_attached_to_run", delete_attachments)
    legacy_webhook = AsyncMock()
    monkeypatch.setattr(svc, "execute_workflow_webhook", legacy_webhook)
    policy_effects = AsyncMock()
    monkeypatch.setattr(svc, "run_terminal_side_effects", policy_effects)

    await svc.clean_up_workflow(
        workflow=workflow,
        workflow_run=workflow_run,
        browser_cleanup_result=WorkflowBrowserCleanupResult(
            browser_state=None,
            tasks=[],
            all_workflow_task_ids=[],
            child_workflow_run_ids=[],
            close_browser_on_completion=True,
        ),
        schedule_credential_fallback_retry=False,
    )

    get_attempts.assert_awaited_once_with(workflow_run.workflow_run_id)
    terminal_hook.assert_awaited_once_with(
        workflow_run_id=workflow_run.workflow_run_id,
        organization_id=workflow_run.organization_id,
        status=workflow_run.status,
        is_final_attempt=True,
    )
    wait_for_uploads.assert_awaited_once_with([])
    save_downloads.assert_awaited_once_with(
        organization_id=workflow_run.organization_id,
        run_id=workflow_run.workflow_run_id,
    )
    delete_attachments.assert_awaited_once_with(run_id=workflow_run.workflow_run_id)
    legacy_webhook.assert_awaited_once_with(workflow_run, None, finalizer="worker_cleanup")
    policy_effects.assert_not_awaited()


@pytest.mark.asyncio
async def test_cleanup_skips_logical_effects_when_policy_attempt_lookup_fails(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    svc = WorkflowService()
    workflow = _execute_workflow(retry_policy=WorkflowRetryPolicy(max_retries=1, retry_on=[{"status": "failed"}]))
    workflow_run = _execute_workflow_run(WorkflowRunStatus.completed)
    get_attempts = AsyncMock(side_effect=RuntimeError("attempt lookup unavailable"))
    monkeypatch.setattr(
        service_module.app,
        "DATABASE",
        SimpleNamespace(
            workflow_run_attempts=SimpleNamespace(get_attempts=get_attempts),
            debug=SimpleNamespace(has_block_run_for_workflow_run=AsyncMock(return_value=False)),
        ),
    )
    terminal_hook = AsyncMock()
    monkeypatch.setattr(
        service_module.app,
        "AGENT_FUNCTION",
        SimpleNamespace(
            on_workflow_run_terminal=terminal_hook,
            is_block_scoped_workflow_run=AsyncMock(return_value=False),
        ),
    )
    monkeypatch.setattr(
        service_module.app,
        "WORKFLOW_CONTEXT_MANAGER",
        SimpleNamespace(remove_workflow_run_context=Mock(), has_workflow_run_context=lambda _workflow_run_id: False),
    )
    monkeypatch.setattr(service_module.skyvern_context, "current", lambda: None)
    monkeypatch.setattr(service_module.app.STORAGE, "save_downloaded_files", AsyncMock())
    monkeypatch.setattr(service_module.app.ARTIFACT_MANAGER, "wait_for_upload_aiotasks", AsyncMock())
    delete_attachments = AsyncMock()
    monkeypatch.setattr(service_module.uploaded_file_service, "delete_files_attached_to_run", delete_attachments)
    legacy_webhook = AsyncMock()
    monkeypatch.setattr(svc, "execute_workflow_webhook", legacy_webhook)
    policy_effects = AsyncMock()
    monkeypatch.setattr(svc, "run_terminal_side_effects", policy_effects)
    credential_fallback = Mock()
    monkeypatch.setattr(svc, "_schedule_credential_fallback_retry", credential_fallback)

    await svc.clean_up_workflow(
        workflow=workflow,
        workflow_run=workflow_run,
        browser_cleanup_result=WorkflowBrowserCleanupResult(
            browser_state=None,
            tasks=[],
            all_workflow_task_ids=[],
            child_workflow_run_ids=[],
            close_browser_on_completion=True,
        ),
    )

    terminal_hook.assert_awaited_once_with(
        workflow_run_id=workflow_run.workflow_run_id,
        organization_id=workflow_run.organization_id,
        status=workflow_run.status,
        is_final_attempt=False,
    )
    delete_attachments.assert_not_awaited()
    legacy_webhook.assert_not_awaited()
    policy_effects.assert_not_awaited()
    credential_fallback.assert_not_called()


@pytest.mark.asyncio
async def test_cleanup_uses_legacy_path_when_policy_run_has_no_attempt_row(monkeypatch: pytest.MonkeyPatch) -> None:
    # Enrollment is the attempt row. A definition with a policy but no row ran the ordinary path, so
    # cleanup owes the legacy webhook and attachment deletion; no recovery sweep can supply them later.
    svc = WorkflowService()
    workflow = _execute_workflow(retry_policy=WorkflowRetryPolicy(max_retries=1, retry_on=[{"status": "failed"}]))
    workflow_run = _execute_workflow_run(WorkflowRunStatus.completed)
    monkeypatch.setattr(
        service_module.app,
        "DATABASE",
        SimpleNamespace(
            workflow_run_attempts=SimpleNamespace(get_attempts=AsyncMock(return_value=[])),
            debug=SimpleNamespace(has_block_run_for_workflow_run=AsyncMock(return_value=False)),
        ),
    )
    terminal_hook = AsyncMock()
    monkeypatch.setattr(
        service_module.app,
        "AGENT_FUNCTION",
        SimpleNamespace(
            on_workflow_run_terminal=terminal_hook,
            is_block_scoped_workflow_run=AsyncMock(return_value=False),
        ),
    )
    monkeypatch.setattr(
        service_module.app,
        "WORKFLOW_CONTEXT_MANAGER",
        SimpleNamespace(remove_workflow_run_context=Mock(), has_workflow_run_context=lambda _workflow_run_id: False),
    )
    monkeypatch.setattr(service_module.skyvern_context, "current", lambda: None)
    monkeypatch.setattr(service_module.app.STORAGE, "save_downloaded_files", AsyncMock())
    monkeypatch.setattr(service_module.app.ARTIFACT_MANAGER, "wait_for_upload_aiotasks", AsyncMock())
    delete_attachments = AsyncMock()
    monkeypatch.setattr(service_module.uploaded_file_service, "delete_files_attached_to_run", delete_attachments)
    legacy_webhook = AsyncMock()
    monkeypatch.setattr(svc, "execute_workflow_webhook", legacy_webhook)
    policy_effects = AsyncMock()
    monkeypatch.setattr(svc, "run_terminal_side_effects", policy_effects)

    await svc.clean_up_workflow(
        workflow=workflow,
        workflow_run=workflow_run,
        browser_cleanup_result=WorkflowBrowserCleanupResult(
            browser_state=None,
            tasks=[],
            all_workflow_task_ids=[],
            child_workflow_run_ids=[],
            close_browser_on_completion=True,
        ),
        schedule_credential_fallback_retry=False,
    )

    terminal_hook.assert_awaited_once_with(
        workflow_run_id=workflow_run.workflow_run_id,
        organization_id=workflow_run.organization_id,
        status=workflow_run.status,
        is_final_attempt=True,
    )
    delete_attachments.assert_awaited_once_with(run_id=workflow_run.workflow_run_id)
    legacy_webhook.assert_awaited_once_with(workflow_run, None, finalizer="worker_cleanup")
    policy_effects.assert_not_awaited()


async def _assert_policy_definition_uses_legacy_cleanup_for_ineligible_run(
    monkeypatch: pytest.MonkeyPatch,
    workflow_run: SimpleNamespace,
    *,
    block_scoped: bool,
) -> None:
    svc = WorkflowService()
    workflow = _execute_workflow(retry_policy=WorkflowRetryPolicy(max_retries=1, retry_on=[{"status": "failed"}]))
    attempt = SimpleNamespace(attempt_number=1, retry_decision=RETRY_DECISION_FINAL)
    terminal_hook = AsyncMock()
    delete_attachments = AsyncMock()
    legacy_webhook = AsyncMock()
    monkeypatch.setattr(
        service_module.app,
        "DATABASE",
        SimpleNamespace(
            workflow_run_attempts=SimpleNamespace(get_attempts=AsyncMock(return_value=[attempt])),
        ),
    )
    monkeypatch.setattr(
        service_module.app,
        "AGENT_FUNCTION",
        SimpleNamespace(
            on_workflow_run_terminal=terminal_hook,
            is_block_scoped_workflow_run=AsyncMock(return_value=block_scoped),
        ),
    )
    monkeypatch.setattr(
        service_module.app,
        "WORKFLOW_CONTEXT_MANAGER",
        SimpleNamespace(remove_workflow_run_context=Mock(), has_workflow_run_context=lambda _workflow_run_id: False),
    )
    monkeypatch.setattr(service_module.skyvern_context, "current", lambda: None)
    monkeypatch.setattr(service_module.app.STORAGE, "save_downloaded_files", AsyncMock())
    monkeypatch.setattr(service_module.app.ARTIFACT_MANAGER, "wait_for_upload_aiotasks", AsyncMock())
    monkeypatch.setattr(service_module.uploaded_file_service, "delete_files_attached_to_run", delete_attachments)
    monkeypatch.setattr(svc, "execute_workflow_webhook", legacy_webhook)
    policy_effects = AsyncMock()
    monkeypatch.setattr(svc, "run_terminal_side_effects", policy_effects)

    await svc.clean_up_workflow(
        workflow=workflow,
        workflow_run=workflow_run,
        browser_cleanup_result=WorkflowBrowserCleanupResult(
            browser_state=None,
            tasks=[],
            all_workflow_task_ids=[],
            child_workflow_run_ids=[],
            close_browser_on_completion=True,
        ),
        schedule_credential_fallback_retry=False,
    )

    terminal_hook.assert_awaited_once_with(
        workflow_run_id=workflow_run.workflow_run_id,
        organization_id=workflow_run.organization_id,
        status=workflow_run.status,
        is_final_attempt=True,
    )
    delete_attachments.assert_awaited_once_with(run_id=workflow_run.workflow_run_id)
    legacy_webhook.assert_awaited_once_with(workflow_run, None, finalizer="worker_cleanup")
    policy_effects.assert_not_awaited()


@pytest.mark.asyncio
async def test_block_scoped_policy_run_uses_legacy_cleanup(monkeypatch: pytest.MonkeyPatch) -> None:
    workflow_run = _execute_workflow_run(WorkflowRunStatus.completed)
    await _assert_policy_definition_uses_legacy_cleanup_for_ineligible_run(
        monkeypatch,
        workflow_run,
        block_scoped=True,
    )


@pytest.mark.asyncio
async def test_child_policy_run_uses_legacy_cleanup(monkeypatch: pytest.MonkeyPatch) -> None:
    workflow_run = _execute_workflow_run(WorkflowRunStatus.completed)
    workflow_run.parent_workflow_run_id = "parent_run"
    await _assert_policy_definition_uses_legacy_cleanup_for_ineligible_run(
        monkeypatch,
        workflow_run,
        block_scoped=False,
    )


@pytest.mark.asyncio
async def test_retry_decision_rejects_non_terminal_status() -> None:
    with pytest.raises(ValueError, match="non-terminal status: running"):
        await service_module.on_terminal_transition(
            SimpleNamespace(workflow_run_id="wr_test"),
            WorkflowRunStatus.running,
            None,
            None,
        )


@pytest.mark.asyncio
async def test_filter_downloaded_files_keeps_identical_redownload_touched_in_current_attempt() -> None:
    svc = WorkflowService()
    attempt_started_at = datetime(2026, 9, 7, 12, 0, 0, tzinfo=UTC)
    files = [
        FileInfo(
            url="https://example.test/old.pdf",
            checksum="old",
            filename="only-attempt-1.pdf",
            modified_at=attempt_started_at.replace(hour=11, minute=59),
            artifact_id="a_old",
        ),
        FileInfo(
            url="https://example.test/report.pdf",
            checksum="same",
            filename="report.pdf",
            modified_at=attempt_started_at,
            artifact_id="a_report",
        ),
    ]

    filtered = svc._filter_downloaded_files_to_attempt(
        files,
        attempt_rows=[SimpleNamespace(attempt_number=2, started_at=attempt_started_at)],
        attempt_number=2,
        artifact_ids={"a_report"},
    )

    assert [file_info.filename for file_info in filtered] == ["report.pdf"]


def test_filter_downloaded_files_bounds_an_unstarted_attempt_by_its_creation() -> None:
    svc = WorkflowService()
    prepared_at = datetime(2026, 9, 7, 12, 0, 0, tzinfo=UTC)
    files = [
        FileInfo(
            url="https://example.test/old.pdf",
            checksum="old",
            filename="attempt-1.pdf",
            modified_at=prepared_at.replace(hour=11),
            artifact_id="a_old",
        ),
        FileInfo(
            url="https://example.test/claimed.pdf",
            checksum="claimed",
            filename="claimed.pdf",
            modified_at=prepared_at.replace(hour=11, minute=30),
            artifact_id="a_claimed",
        ),
        FileInfo(url="https://example.test/unknown.pdf", checksum="unknown", filename="unknown.pdf"),
    ]
    attempt_rows = [
        SimpleNamespace(
            attempt_number=1, started_at=prepared_at.replace(hour=10), created_at=prepared_at.replace(hour=9)
        ),
        SimpleNamespace(attempt_number=2, started_at=None, created_at=prepared_at),
    ]

    filtered = svc._filter_downloaded_files_to_attempt(
        files, attempt_rows=attempt_rows, attempt_number=2, artifact_ids={"a_claimed"}
    )

    assert [file_info.filename for file_info in filtered] == ["claimed.pdf"]
    # The live attempt keeps the permissive rule for a file whose timestamp is unknown.
    live = svc._filter_downloaded_files_to_attempt(
        files[2:], attempt_rows=attempt_rows[:1], attempt_number=1, artifact_ids=set()
    )
    assert [file_info.filename for file_info in live] == ["unknown.pdf"]


@pytest.mark.asyncio
async def test_execute_workflow_persists_profile_when_only_finally_block_fails(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    workflow = _execute_workflow()
    workflow.workflow_definition.finally_block_label = "cleanup"
    failed_run = _execute_workflow_run(WorkflowRunStatus.failed)
    failed_run.failure_reason = "cleanup failed"
    order: list[str] = []

    svc = WorkflowService()
    _patch_execute_workflow_deps(monkeypatch, svc, workflow, _execute_workflow_run(WorkflowRunStatus.running))
    monkeypatch.setattr(
        svc,
        "_execute_finally_block_if_configured",
        AsyncMock(
            return_value=(
                SimpleNamespace(label="cleanup", block_type="cloud_storage", continue_on_failure=False),
                SimpleNamespace(
                    status=BlockStatus.failed,
                    failure_reason="cleanup failed",
                    output_parameter_value=None,
                    sign_in_form_visible=False,
                ),
            )
        ),
    )
    apply_finally_result = AsyncMock(
        return_value=(failed_run, WorkflowRunStatus.failed, failed_run.failure_reason),
    )
    monkeypatch.setattr(svc, "_apply_finally_block_result", apply_finally_result)
    clean_up_browser = _patch_browser_cleanup(monkeypatch, svc, order)
    monkeypatch.setattr(
        svc,
        "_persist_workflow_browser_session_if_needed",
        AsyncMock(side_effect=lambda **_kwargs: order.append("store")),
    )
    _patch_finalize(monkeypatch, svc, order, failed_run)
    monkeypatch.setattr(svc, "persist_video_data", AsyncMock(side_effect=lambda *a, **k: order.append("video")))
    monkeypatch.setattr(svc, "execute_workflow_webhook", AsyncMock(side_effect=lambda *a, **k: order.append("webhook")))

    result = await _run_execute_workflow(svc)

    assert result is failed_run
    assert order == ["teardown", "store", "finalize", "video", "webhook"]
    clean_up_browser.assert_awaited_once()
    svc._persist_workflow_browser_session_if_needed.assert_awaited_once()
    assert apply_finally_result.await_args is not None
    assert apply_finally_result.await_args.kwargs["pre_finally_status"] == WorkflowRunStatus.running
    assert apply_finally_result.await_args.kwargs["defer_status_write"] is True


@pytest.mark.asyncio
async def test_execute_workflow_defers_finally_failure_status_until_after_writeback(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    workflow = _execute_workflow()
    workflow.persist_browser_session = False
    workflow.workflow_definition.finally_block_label = "cleanup"
    running_run = _execute_workflow_run(WorkflowRunStatus.running)
    failed_run = _execute_workflow_run(WorkflowRunStatus.failed)
    failed_run.failure_reason = "cloud_storage block failed. failure reason: upload failed"
    finally_block = SimpleNamespace(
        block_type="cloud_storage",
        label="cleanup",
        continue_on_failure=False,
    )
    finally_result = SimpleNamespace(
        status=BlockStatus.failed,
        failure_reason="upload failed",
        output_parameter_value=None,
        sign_in_form_visible=False,
    )

    svc = WorkflowService()
    _patch_execute_workflow_deps(monkeypatch, svc, workflow, running_run)
    monkeypatch.setattr(
        svc,
        "_execute_finally_block_if_configured",
        AsyncMock(return_value=(finally_block, finally_result)),
    )
    status_write = AsyncMock(side_effect=RuntimeError("status write must remain deferred"))
    monkeypatch.setattr(svc, "_update_workflow_run_status_if_not_final", status_write)
    order: list[str] = []

    def tag_signed_out(
        _workflow_run_id: str, _status: WorkflowRunStatus, category: list[dict] | None
    ) -> list[dict] | None:
        order.append("tag")
        return category

    monkeypatch.setattr(svc, "_with_saved_profile_signed_out", tag_signed_out)
    _patch_browser_cleanup(monkeypatch, svc, order)
    persist_browser_session = AsyncMock(side_effect=lambda **_kwargs: order.append("store"))
    monkeypatch.setattr(
        svc,
        "_persist_workflow_browser_session_if_needed",
        persist_browser_session,
    )

    async def finalize_side_effect(**_kwargs: object) -> SimpleNamespace:
        order.append("finalize")
        return failed_run

    finalize = AsyncMock(side_effect=finalize_side_effect)
    monkeypatch.setattr(svc, "_finalize_workflow_run_status", finalize)
    clean_up = AsyncMock()
    monkeypatch.setattr(svc, "clean_up_workflow", clean_up)

    result = await _run_execute_workflow(svc)

    assert result is failed_run
    # The saved-profile tag reads the run's open browser, so it is applied before teardown.
    assert order == ["tag", "teardown", "store", "finalize"]
    status_write.assert_not_awaited()
    persist_browser_session.assert_awaited_once()
    assert persist_browser_session.await_args is not None
    assert persist_browser_session.await_args.kwargs["workflow_run_status"] == WorkflowRunStatus.completed
    finalize.assert_awaited_once()
    assert finalize.await_args is not None
    assert finalize.await_args.kwargs["pre_finally_status"] == WorkflowRunStatus.failed
    assert (
        finalize.await_args.kwargs["pre_finally_failure_reason"]
        == "cloud_storage block failed. failure reason: upload failed"
    )
    assert finalize.await_args.kwargs["pre_finally_failure_category"]
    clean_up.assert_awaited_once()
    assert clean_up.await_args is not None
    assert clean_up.await_args.kwargs["browser_persistence_status"] == WorkflowRunStatus.completed
    assert clean_up.await_args.kwargs["schedule_credential_fallback_retry"] is False


@pytest.mark.asyncio
async def test_execute_workflow_skips_profile_persistence_when_timeout_precedes_body_status_capture(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    workflow = _execute_workflow()
    running_run = _execute_workflow_run(WorkflowRunStatus.running)
    timed_out_run = _execute_workflow_run(WorkflowRunStatus.timed_out)
    timed_out_run.failure_reason = "Workflow exceeded its elapsed-time limit."

    svc = WorkflowService()
    _patch_execute_workflow_deps(monkeypatch, svc, workflow, running_run)

    async def slow_refresh(**_kwargs: object) -> SimpleNamespace:
        await asyncio.sleep(0.05)
        return running_run

    service_module.app.DATABASE.workflow_runs.get_workflow_run = AsyncMock(side_effect=slow_refresh)
    timeout_seconds = iter([10.0, 0.01])
    monkeypatch.setattr(
        service_module,
        "_get_workflow_run_max_elapsed_timeout_seconds",
        lambda _workflow_run: next(timeout_seconds),
    )
    monkeypatch.setattr(
        svc,
        "_shield_post_run_elapsed_timeout",
        AsyncMock(
            return_value=(
                timed_out_run,
                WorkflowRunStatus.timed_out,
                timed_out_run.failure_reason,
            )
        ),
    )
    prefinal_browser_cleanup = AsyncMock()
    persist_browser_session = AsyncMock()
    monkeypatch.setattr(svc, "_clean_up_workflow_browser", prefinal_browser_cleanup)
    monkeypatch.setattr(svc, "_persist_workflow_browser_session_if_needed", persist_browser_session)
    monkeypatch.setattr(svc, "_finalize_workflow_run_status", AsyncMock(return_value=timed_out_run))
    monkeypatch.setattr(svc, "persist_video_data", AsyncMock())
    monkeypatch.setattr(svc, "clean_up_workflow", AsyncMock())
    monkeypatch.setattr(svc, "execute_workflow_webhook", AsyncMock())

    result = await _run_execute_workflow(svc)

    assert result is timed_out_run
    prefinal_browser_cleanup.assert_not_awaited()
    persist_browser_session.assert_not_awaited()


@pytest.mark.asyncio
async def test_execute_workflow_rechecks_terminal_winner_before_healthy_writeback(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    workflow = _execute_workflow()
    running_run = _execute_workflow_run(WorkflowRunStatus.running)
    timed_out_run = _execute_workflow_run(WorkflowRunStatus.timed_out)
    timed_out_run.failure_reason = "Workflow exceeded its elapsed-time limit."

    svc = WorkflowService()
    _patch_execute_workflow_deps(monkeypatch, svc, workflow, running_run)
    initial_run = svc.get_workflow_run.return_value
    monkeypatch.setattr(svc, "get_workflow_run", AsyncMock(side_effect=[initial_run, timed_out_run]))
    prefinal_browser_cleanup = AsyncMock()
    persist_browser_session = AsyncMock()
    monkeypatch.setattr(svc, "_clean_up_workflow_browser", prefinal_browser_cleanup)
    monkeypatch.setattr(svc, "_persist_workflow_browser_session_if_needed", persist_browser_session)
    finalize = AsyncMock(return_value=timed_out_run)
    monkeypatch.setattr(svc, "_finalize_workflow_run_status", finalize)
    clean_up = AsyncMock()
    monkeypatch.setattr(svc, "clean_up_workflow", clean_up)

    result = await _run_execute_workflow(svc)

    assert result is timed_out_run
    prefinal_browser_cleanup.assert_not_awaited()
    persist_browser_session.assert_not_awaited()
    finalize.assert_awaited_once()
    assert finalize.await_args is not None
    assert finalize.await_args.kwargs["workflow"] is workflow
    clean_up.assert_awaited_once()
    assert clean_up.await_args is not None
    assert clean_up.await_args.kwargs["browser_persistence_status"] == WorkflowRunStatus.timed_out


@pytest.mark.asyncio
async def test_execute_workflow_retries_write_back_when_pre_final_persist_fails(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    workflow = _execute_workflow()
    completed_run = _execute_workflow_run(WorkflowRunStatus.completed)
    order: list[str] = []
    persist_calls = {"n": 0}

    async def persist_side_effect(**_kwargs: object) -> None:
        persist_calls["n"] += 1
        if persist_calls["n"] == 1:
            order.append("store_fail")
            raise RuntimeError("storage down")
        order.append("store_retry")

    svc = WorkflowService()
    _patch_execute_workflow_deps(monkeypatch, svc, workflow, _execute_workflow_run(WorkflowRunStatus.running))
    _patch_browser_cleanup(monkeypatch, svc, order)
    monkeypatch.setattr(svc, "_persist_workflow_browser_session_if_needed", AsyncMock(side_effect=persist_side_effect))
    _patch_finalize(monkeypatch, svc, order, completed_run)
    monkeypatch.setattr(svc, "persist_video_data", AsyncMock(side_effect=lambda *a, **k: order.append("video")))
    monkeypatch.setattr(svc, "execute_workflow_webhook", AsyncMock(side_effect=lambda *a, **k: order.append("webhook")))

    result = await _run_execute_workflow(svc)

    assert result is completed_run
    # Sequential dependents remain blocked until the write-back retry has also completed.
    assert order == ["teardown", "store_fail", "store_retry", "finalize", "video", "webhook"]
    assert svc._persist_workflow_browser_session_if_needed.await_count == 2


@pytest.mark.asyncio
async def test_execute_workflow_exhausts_write_back_before_final_status(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    workflow = _execute_workflow()
    completed_run = _execute_workflow_run(WorkflowRunStatus.completed)
    order: list[str] = []
    persist_calls = {"n": 0}

    async def persist_side_effect(**_kwargs: object) -> None:
        persist_calls["n"] += 1
        order.append(f"store_fail_{persist_calls['n']}")
        raise RuntimeError("storage down")

    svc = WorkflowService()
    _patch_execute_workflow_deps(monkeypatch, svc, workflow, _execute_workflow_run(WorkflowRunStatus.running))
    _patch_browser_cleanup(monkeypatch, svc, order)
    monkeypatch.setattr(svc, "_persist_workflow_browser_session_if_needed", AsyncMock(side_effect=persist_side_effect))
    _patch_finalize(monkeypatch, svc, order, completed_run)
    monkeypatch.setattr(svc, "persist_video_data", AsyncMock(side_effect=lambda *a, **k: order.append("video")))
    monkeypatch.setattr(svc, "execute_workflow_webhook", AsyncMock(side_effect=lambda *a, **k: order.append("webhook")))

    result = await _run_execute_workflow(svc)

    assert result is completed_run
    assert order == ["teardown", "store_fail_1", "store_fail_2", "finalize", "video", "webhook"]
    assert svc._persist_workflow_browser_session_if_needed.await_count == 2


@pytest.mark.asyncio
async def test_execute_workflow_does_not_prestore_blob_for_failed_canceled_or_timed_out_runs(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    for status in (WorkflowRunStatus.failed, WorkflowRunStatus.canceled, WorkflowRunStatus.timed_out):
        workflow = _execute_workflow()
        terminal_run = _execute_workflow_run(status)
        order: list[str] = []

        svc = WorkflowService()
        _patch_execute_workflow_deps(monkeypatch, svc, workflow, terminal_run)
        _patch_browser_cleanup(monkeypatch, svc, order)
        monkeypatch.setattr(
            service_module.app.STORAGE,
            "store_browser_profile",
            AsyncMock(side_effect=lambda *a, **k: order.append("store_profile")),
        )
        monkeypatch.setattr(service_module.app.STORAGE, "store_browser_session", AsyncMock())
        _patch_finalize(monkeypatch, svc, order, terminal_run)
        monkeypatch.setattr(svc, "persist_video_data", AsyncMock(side_effect=lambda *a, **k: order.append("video")))
        monkeypatch.setattr(svc, "execute_workflow_webhook", AsyncMock())

        result = await _run_execute_workflow(svc)

        assert result is terminal_run
        assert order[:2] == ["finalize", "teardown"]
        assert "store_profile" not in order


@pytest.mark.asyncio
async def test_execute_workflow_non_persist_workflow_writes_sink_before_final_status(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    workflow = _execute_workflow()
    workflow.persist_browser_session = False
    completed_run = _execute_workflow_run(WorkflowRunStatus.completed)
    order: list[str] = []

    svc = WorkflowService()
    _patch_execute_workflow_deps(monkeypatch, svc, workflow, _execute_workflow_run(WorkflowRunStatus.running))
    clean_up_browser = _patch_browser_cleanup(monkeypatch, svc, order)
    persist_browser_session = AsyncMock(side_effect=lambda **_kwargs: order.append("store"))
    monkeypatch.setattr(svc, "_persist_workflow_browser_session_if_needed", persist_browser_session)
    _patch_finalize(monkeypatch, svc, order, completed_run)
    monkeypatch.setattr(svc, "persist_video_data", AsyncMock(side_effect=lambda *a, **k: order.append("video")))
    monkeypatch.setattr(svc, "execute_workflow_webhook", AsyncMock())

    result = await _run_execute_workflow(svc)

    assert result is completed_run
    assert order[:3] == ["teardown", "store", "finalize"]
    clean_up_browser.assert_awaited_once()
    persist_browser_session.assert_awaited_once()
    assert persist_browser_session.await_args is not None
    assert persist_browser_session.await_args.kwargs["workflow_run_status"] == WorkflowRunStatus.completed


@pytest.mark.asyncio
async def test_execute_workflow_finalizes_when_pre_status_browser_cleanup_fails(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    workflow = _execute_workflow()
    completed_run = _execute_workflow_run(WorkflowRunStatus.completed)
    order: list[str] = []

    def cleanup_side_effect(**_kwargs: object) -> WorkflowBrowserCleanupResult:
        if "teardown_error" not in order:
            order.append("teardown_error")
            raise RuntimeError("cleanup failed")
        order.append("teardown_cleanup")
        return _browser_cleanup_result()

    svc = WorkflowService()
    _patch_execute_workflow_deps(monkeypatch, svc, workflow, _execute_workflow_run(WorkflowRunStatus.running))
    monkeypatch.setattr(svc, "_clean_up_workflow_browser", AsyncMock(side_effect=cleanup_side_effect))
    monkeypatch.setattr(
        svc,
        "_persist_workflow_browser_session_if_needed",
        AsyncMock(side_effect=lambda **_kwargs: order.append("persist_helper")),
    )
    _patch_finalize(monkeypatch, svc, order, completed_run)
    monkeypatch.setattr(svc, "persist_video_data", AsyncMock())
    monkeypatch.setattr(svc, "execute_workflow_webhook", AsyncMock())

    result = await _run_execute_workflow(svc)

    assert result is completed_run
    assert order[:4] == ["teardown_error", "teardown_cleanup", "persist_helper", "finalize"]
    assert svc._clean_up_workflow_browser.await_count == 2


def _patch_session_backed_run(monkeypatch: pytest.MonkeyPatch, svc: WorkflowService) -> AsyncMock:
    monkeypatch.setattr(
        service_module.skyvern_context,
        "ensure_context",
        lambda: SkyvernContext(
            browser_session_id=None,
            browser_session_runnable_id=None,
            browser_session_runnable_generation_id=None,
        ),
    )
    monkeypatch.setattr(app.PERSISTENT_SESSIONS_MANAGER, "begin_session", AsyncMock(return_value="lease_1"))
    close_session = AsyncMock()
    monkeypatch.setattr(app.PERSISTENT_SESSIONS_MANAGER, "close_session", close_session)
    monkeypatch.setattr(svc, "persist_video_data", AsyncMock())
    monkeypatch.setattr(svc, "_persist_workflow_browser_session_if_needed", AsyncMock())
    monkeypatch.setattr(svc, "execute_workflow_webhook", AsyncMock())
    return close_session


@pytest.mark.asyncio
async def test_execute_workflow_does_not_close_caller_supplied_session(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    workflow = _execute_workflow()
    completed_run = _execute_workflow_run(WorkflowRunStatus.completed)
    order: list[str] = []

    svc = WorkflowService()
    _patch_execute_workflow_deps(monkeypatch, svc, workflow, _execute_workflow_run(WorkflowRunStatus.running))
    close_session = _patch_session_backed_run(monkeypatch, svc)
    _patch_browser_cleanup(monkeypatch, svc, order)
    _patch_finalize(monkeypatch, svc, order, completed_run)

    result = await _run_execute_workflow(svc, browser_session_id="pbs_caller")

    assert result is completed_run
    close_session.assert_not_awaited()


def _patch_human_interaction_session(monkeypatch: pytest.MonkeyPatch, svc: WorkflowService) -> AsyncMock:
    close_session = _patch_session_backed_run(monkeypatch, svc)
    # The fixture run carries a browser profile; the auto-create only runs for managed profiles.
    monkeypatch.setattr(svc, "_browser_profile_is_managed", AsyncMock(return_value=True))
    monkeypatch.setattr(
        svc,
        "auto_create_browser_session_if_needed",
        AsyncMock(return_value=SimpleNamespace(persistent_browser_session_id="pbs_human")),
    )
    return close_session


@pytest.mark.asyncio
@pytest.mark.parametrize("terminal_status", [WorkflowRunStatus.completed, WorkflowRunStatus.failed])
async def test_execute_workflow_closes_auto_created_human_interaction_session(
    monkeypatch: pytest.MonkeyPatch,
    terminal_status: WorkflowRunStatus,
) -> None:
    workflow = _execute_workflow()
    terminal_run = _execute_workflow_run(terminal_status)
    order: list[str] = []

    svc = WorkflowService()
    _patch_execute_workflow_deps(monkeypatch, svc, workflow, _execute_workflow_run(WorkflowRunStatus.running))
    close_session = _patch_human_interaction_session(monkeypatch, svc)
    _patch_browser_cleanup(monkeypatch, svc, order)
    _patch_finalize(monkeypatch, svc, order, terminal_run)

    result = await _run_execute_workflow(svc)

    assert result is terminal_run
    close_session.assert_awaited_once_with("o_test", "pbs_human", reason=BrowserSessionCloseReason.run_ended)


@pytest.mark.asyncio
async def test_execute_workflow_completes_cleanup_when_owned_session_close_fails(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    workflow = _execute_workflow()
    completed_run = _execute_workflow_run(WorkflowRunStatus.completed)
    order: list[str] = []

    svc = WorkflowService()
    _patch_execute_workflow_deps(monkeypatch, svc, workflow, _execute_workflow_run(WorkflowRunStatus.running))
    close_session = _patch_human_interaction_session(monkeypatch, svc)
    close_session.side_effect = RuntimeError("close failed")
    _patch_browser_cleanup(monkeypatch, svc, order)
    _patch_finalize(monkeypatch, svc, order, completed_run)

    result = await _run_execute_workflow(svc)

    assert result is completed_run
    close_session.assert_awaited_once()
    svc.execute_workflow_webhook.assert_awaited_once()


@pytest.mark.asyncio
async def test_execute_workflow_closes_owned_session_when_begin_session_fails(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    workflow = _execute_workflow()
    failed_run = _execute_workflow_run(WorkflowRunStatus.failed)
    order: list[str] = []

    svc = WorkflowService()
    _patch_execute_workflow_deps(monkeypatch, svc, workflow, _execute_workflow_run(WorkflowRunStatus.running))
    close_session = _patch_human_interaction_session(monkeypatch, svc)
    monkeypatch.setattr(
        app.PERSISTENT_SESSIONS_MANAGER, "begin_session", AsyncMock(side_effect=RuntimeError("begin failed"))
    )
    monkeypatch.setattr(svc, "mark_workflow_run_as_failed", AsyncMock(return_value=failed_run))
    _patch_browser_cleanup(monkeypatch, svc, order)

    result = await _run_execute_workflow(svc)

    assert result is failed_run
    close_session.assert_awaited_once_with("o_test", "pbs_human", reason=BrowserSessionCloseReason.run_ended)


@pytest.mark.asyncio
@pytest.mark.parametrize(
    ("created_by", "bound", "ended_after_run_created", "expected_category", "expected_reason_code"),
    [
        pytest.param(
            "api", False, False, "BROWSER_SESSION_EXPIRED", "browser_session_expired_before_run", id="customer"
        ),
        pytest.param(
            None, False, False, "BROWSER_SESSION_EXPIRED", "browser_session_expired_before_run", id="legacy-null"
        ),
        pytest.param(None, True, False, "BROWSER_ERROR", "browser_session_closed", id="reuse-bound-session"),
        pytest.param("copilot", False, False, "BROWSER_ERROR", "browser_session_closed", id="copilot-session"),
        pytest.param("api", False, True, "BROWSER_ERROR", "browser_session_closed", id="expired-while-queued"),
    ],
)
async def test_lease_of_an_expired_session_is_the_callers_only_when_sent_after_it_ended(
    monkeypatch: pytest.MonkeyPatch,
    created_by: str | None,
    bound: bool,
    ended_after_run_created: bool,
    expected_category: str,
    expected_reason_code: str,
) -> None:
    workflow = _execute_workflow()
    failed_run = _execute_workflow_run(WorkflowRunStatus.failed)
    svc = WorkflowService()
    _patch_execute_workflow_deps(monkeypatch, svc, workflow, _execute_workflow_run(WorkflowRunStatus.running))
    _patch_session_backed_run(monkeypatch, svc)
    run_created_at = _execute_workflow_run(WorkflowRunStatus.running).created_at
    ended_at = run_created_at + timedelta(minutes=1) if ended_after_run_created else run_created_at - timedelta(hours=6)
    expired = BrowserSessionExpired(
        "pbs_caller",
        started_at=ended_at - timedelta(minutes=240),
        ended_at=ended_at,
        timeout_minutes=240,
        created_by=created_by,
        bound=bound,
    )
    monkeypatch.setattr(app.PERSISTENT_SESSIONS_MANAGER, "begin_session", AsyncMock(side_effect=expired))
    mark_failed = AsyncMock(return_value=failed_run)
    monkeypatch.setattr(svc, "mark_workflow_run_as_failed", mark_failed)
    _patch_browser_cleanup(monkeypatch, svc, [])

    assert await _run_execute_workflow(svc, browser_session_id="pbs_caller") is failed_run

    failure_category = mark_failed.await_args.kwargs["failure_category"]
    failure_reason = mark_failed.await_args.kwargs["failure_reason"]
    assert (failure_category[0]["category"], failure_category[0]["reason_code"]) == (
        expected_category,
        expected_reason_code,
    )
    assert "pbs_caller" in failure_reason


@pytest.mark.asyncio
async def test_owned_session_close_deferred_while_child_runs_active(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    workflow = _execute_workflow()
    completed_run = _execute_workflow_run(WorkflowRunStatus.completed)
    order: list[str] = []

    svc = WorkflowService()
    _patch_execute_workflow_deps(monkeypatch, svc, workflow, _execute_workflow_run(WorkflowRunStatus.running))
    close_session = _patch_human_interaction_session(monkeypatch, svc)
    cleanup_result = _browser_cleanup_result()
    cleanup_result.child_workflow_run_ids = ["wr_child"]
    monkeypatch.setattr(
        svc,
        "_clean_up_workflow_browser",
        AsyncMock(side_effect=lambda **_kwargs: order.append("teardown") or cleanup_result),
    )
    monkeypatch.setattr(
        service_module.app.DATABASE.workflow_runs,
        "get_workflow_runs_by_parent_workflow_run_id",
        AsyncMock(return_value=[_execute_workflow_run(WorkflowRunStatus.running)]),
    )
    _patch_finalize(monkeypatch, svc, order, completed_run)

    result = await _run_execute_workflow(svc)

    assert result is completed_run
    close_session.assert_not_awaited()


@pytest.mark.asyncio
async def test_owned_session_closes_when_child_runs_are_all_terminal(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    workflow = _execute_workflow()
    completed_run = _execute_workflow_run(WorkflowRunStatus.completed)
    order: list[str] = []

    svc = WorkflowService()
    _patch_execute_workflow_deps(monkeypatch, svc, workflow, _execute_workflow_run(WorkflowRunStatus.running))
    close_session = _patch_human_interaction_session(monkeypatch, svc)
    cleanup_result = _browser_cleanup_result()
    cleanup_result.child_workflow_run_ids = ["wr_child"]
    monkeypatch.setattr(
        svc,
        "_clean_up_workflow_browser",
        AsyncMock(side_effect=lambda **_kwargs: order.append("teardown") or cleanup_result),
    )
    monkeypatch.setattr(
        service_module.app.DATABASE.workflow_runs,
        "get_workflow_runs_by_parent_workflow_run_id",
        AsyncMock(return_value=[_execute_workflow_run(WorkflowRunStatus.completed)]),
    )
    _patch_finalize(monkeypatch, svc, order, completed_run)

    result = await _run_execute_workflow(svc)

    assert result is completed_run
    close_session.assert_awaited_once_with("o_test", "pbs_human", reason=BrowserSessionCloseReason.run_ended)


@pytest.mark.asyncio
async def test_execute_workflow_closes_owned_session_when_cancelled_mid_run(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    workflow = _execute_workflow()
    order: list[str] = []

    svc = WorkflowService()
    _patch_execute_workflow_deps(monkeypatch, svc, workflow, _execute_workflow_run(WorkflowRunStatus.running))
    close_session = _patch_human_interaction_session(monkeypatch, svc)
    monkeypatch.setattr(svc, "_execute_workflow_blocks", AsyncMock(side_effect=asyncio.CancelledError()))
    _patch_browser_cleanup(monkeypatch, svc, order)
    _patch_finalize(monkeypatch, svc, order, _execute_workflow_run(WorkflowRunStatus.canceled))

    with pytest.raises(asyncio.CancelledError):
        await _run_execute_workflow(svc)

    close_session.assert_awaited_once_with("o_test", "pbs_human", reason=BrowserSessionCloseReason.run_ended)


@pytest.mark.asyncio
async def test_clean_up_workflow_closes_owned_session_when_webhook_raises(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    workflow = _execute_workflow()
    completed_run = _execute_workflow_run(WorkflowRunStatus.completed)

    svc = WorkflowService()
    _patch_execute_workflow_deps(monkeypatch, svc, workflow, _execute_workflow_run(WorkflowRunStatus.running))
    close_session = _patch_session_backed_run(monkeypatch, svc)
    monkeypatch.setattr(svc, "execute_workflow_webhook", AsyncMock(side_effect=RuntimeError("webhook down")))

    with pytest.raises(RuntimeError, match="webhook down"):
        await svc.clean_up_workflow(
            workflow=workflow,
            workflow_run=completed_run,
            api_key=None,
            browser_session_id="pbs_human",
            browser_cleanup_result=_browser_cleanup_result(),
            owned_browser_session_id="pbs_human",
        )

    close_session.assert_awaited_once_with("o_test", "pbs_human", reason=BrowserSessionCloseReason.run_ended)


@pytest_asyncio.fixture
async def forced_session_setup(
    monkeypatch: pytest.MonkeyPatch,
    sqlite_engine: AsyncEngine,
) -> AsyncIterator[tuple[AgentDB, WorkflowService, Workflow]]:
    database = AgentDB("sqlite+aiosqlite://", db_engine=sqlite_engine)
    await database.organizations.create_organization("Test", organization_id="o_test")
    async with database.Session() as session:
        session.add(
            WorkflowModel(
                workflow_id="wf_1",
                workflow_permanent_id="wpid_test",
                organization_id="o_test",
                title="Workflow",
                workflow_definition={
                    "parameters": [],
                    "blocks": [],
                    "retry_policy": {"retry_on": [{"status": "failed"}], "delay_seconds": 0},
                },
            )
        )
        await session.commit()
    workflow = await database.workflows.get_workflow("wf_1", organization_id="o_test")
    assert workflow is not None
    monkeypatch.setattr(app, "DATABASE", database)
    monkeypatch.setattr(app.WORKFLOW_CONTEXT_MANAGER, "remove_workflow_run_context", Mock())
    monkeypatch.setattr(app.AGENT_FUNCTION, "is_block_scoped_workflow_run", AsyncMock(return_value=False))
    monkeypatch.setattr(
        app.EXPERIMENTATION_PROVIDER,
        "is_feature_enabled_cached",
        AsyncMock(side_effect=lambda flag, *args, **kwargs: flag == "FORCE_BROWSER_SESSION"),
    )

    async def create_session(**kwargs: Any) -> SimpleNamespace:
        async with database.Session() as session:
            if await session.get(PersistentBrowserSessionModel, "pbs_forced") is None:
                session.add(
                    PersistentBrowserSessionModel(
                        persistent_browser_session_id="pbs_forced",
                        organization_id="o_test",
                        status="running",
                        runnable_type="forced_workflow_run",
                    )
                )
                await session.commit()
        run = await database.workflow_runs.get_workflow_run(kwargs["workflow_run_id"], "o_test")
        if run is not None and (run.reuse_bound_key or "").startswith("off:force_pending:"):
            await database.workflow_runs.update_workflow_run(kwargs["workflow_run_id"], browser_session_id="pbs_forced")
        return SimpleNamespace(persistent_browser_session_id="pbs_forced")

    monkeypatch.setattr(app.PERSISTENT_SESSIONS_MANAGER, "create_session", AsyncMock(side_effect=create_session))
    service = WorkflowService()
    monkeypatch.setattr(service, "_resolve_managed_browser_profile_for_run_request", AsyncMock(return_value=None))
    yield database, service, workflow
    while service._background_tasks:
        await asyncio.gather(*list(service._background_tasks))


@pytest.mark.asyncio
@pytest.mark.parametrize("forced_session", [True, False])
async def test_block_scoped_setup_does_not_enroll_a_retry_attempt(
    monkeypatch: pytest.MonkeyPatch,
    forced_session_setup: tuple[AgentDB, WorkflowService, Workflow],
    forced_session: bool,
) -> None:
    """A block run creates its block-run rows only after setup, so enrolment must trust the
    caller's block-scoped intent; a policy definition alone must not enroll it."""
    database, service, workflow = forced_session_setup
    monkeypatch.setattr(
        app.EXPERIMENTATION_PROVIDER,
        "is_feature_enabled_cached",
        AsyncMock(side_effect=lambda flag, *args, **kwargs: forced_session and flag == "FORCE_BROWSER_SESSION"),
    )
    block_run = await service.create_workflow_run(
        workflow_request=WorkflowRequestBody(),
        workflow_permanent_id="wpid_test",
        workflow_id="wf_1",
        organization_id="o_test",
        workflow=workflow,
        block_scoped=True,
    )
    full_run = await service.create_workflow_run(
        workflow_request=WorkflowRequestBody(),
        workflow_permanent_id="wpid_test",
        workflow_id="wf_1",
        organization_id="o_test",
        workflow=workflow,
    )
    assert await database.workflow_run_attempts.get_attempts(block_run.workflow_run_id) == []
    enrolled = await database.workflow_run_attempts.get_attempts(full_run.workflow_run_id)
    assert [attempt.attempt_number for attempt in enrolled] == [1]


@pytest_asyncio.fixture(params=["sequential", "browser_reuse"])
async def retry_gate_setup(
    monkeypatch: pytest.MonkeyPatch,
    forced_session_setup: tuple[AgentDB, WorkflowService, Workflow],
    request: pytest.FixtureRequest,
) -> tuple[AgentDB, WorkflowService, WorkflowRun]:
    database, service, _ = forced_session_setup
    async with database.Session() as session:
        workflow_model = await session.get(WorkflowModel, "wf_1")
        assert workflow_model is not None
        workflow_model.run_sequentially = request.param == "sequential"
        workflow_model.reuse_browser_session = request.param == "browser_reuse"
        workflow_model.workflow_definition = {
            "parameters": [],
            "blocks": [],
            "retry_policy": {"retry_on": [{"status": "timed_out"}], "delay_seconds": 0, "max_retries": 1},
        }
        await session.commit()
    workflow = await database.workflows.get_workflow("wf_1", organization_id="o_test")
    assert workflow is not None
    monkeypatch.setattr(app.EXPERIMENTATION_PROVIDER, "is_feature_enabled_cached", AsyncMock(return_value=False))
    monkeypatch.setattr(app, "WORKFLOW_SERVICE", service)
    monkeypatch.setattr(service, "_run_interim_side_effects_with_retries", AsyncMock(return_value="released"))
    monkeypatch.setattr(service, "_run_terminal_side_effects_with_retries", AsyncMock(return_value="released"))
    run = await service.create_workflow_run(
        workflow_request=WorkflowRequestBody(start_fresh_browser=False),
        workflow_permanent_id="wpid_test",
        workflow_id="wf_1",
        organization_id="o_test",
        workflow=workflow,
        max_elapsed_time_minutes=1,
    )
    expired_start = datetime.now(UTC).replace(tzinfo=None) - timedelta(minutes=2)
    run = await database.workflow_runs.update_workflow_run(
        run.workflow_run_id, status=WorkflowRunStatus.running, started_at=expired_start
    )
    async with database.Session() as session:
        attempt = await session.get(WorkflowRunAttemptModel, (run.workflow_run_id, 1))
        assert attempt is not None
        attempt.started_at = expired_start
        await session.commit()

    async def mark_timed_out(workflow_run_id: str, failure_reason: str) -> WorkflowRun:
        return await database.workflow_runs.update_workflow_run(
            workflow_run_id,
            status=WorkflowRunStatus.timed_out,
            failure_reason=failure_reason,
            finished_at=datetime.now(UTC).replace(tzinfo=None),
        )

    monkeypatch.setattr(service, "mark_workflow_run_as_timed_out", mark_timed_out)
    return database, service, run


@pytest.mark.asyncio
@pytest.mark.parametrize("blocked", [False, True])
@pytest.mark.parametrize("escaped_exception", [False, True])
async def test_timed_out_retry_gets_fresh_admission_budget(
    monkeypatch: pytest.MonkeyPatch,
    retry_gate_setup: tuple[AgentDB, WorkflowService, WorkflowRun],
    blocked: bool,
    escaped_exception: bool,
) -> None:
    database, service, run = retry_gate_setup
    executed_attempts: list[int] = []
    admission_budgets: list[float] = []
    gate_sleeps: list[float] = []

    async def execute_attempt(*, attempt_number: int, **kwargs: Any) -> WorkflowRun:
        kwargs["on_execution_authority"](attempt_number, kwargs["dispatch_claim_started_at"])
        executed_attempts.append(attempt_number)
        result = await database.workflow_runs.update_workflow_run(
            run.workflow_run_id,
            status=WorkflowRunStatus.timed_out if attempt_number == 1 else WorkflowRunStatus.completed,
        )
        if attempt_number == 1 and escaped_exception:
            raise RuntimeError("attempt timed out")
        return result

    admission_timeout: asyncio.Timeout | None = None

    def record_admission_timeout(seconds: float) -> asyncio.Timeout:
        nonlocal admission_timeout
        admission_budgets.append(seconds)
        admission_timeout = asyncio.timeout(None)
        return admission_timeout

    async def expire_timeout_at_gate_sleep(seconds: float) -> None:
        if seconds > 0:
            gate_sleeps.append(seconds)
            assert admission_timeout is not None
            # Cancel at the gate sleep, after the database read and session cleanup complete.
            admission_timeout.reschedule(asyncio.get_running_loop().time())
        await asyncio.sleep(0)

    monkeypatch.setattr(service, "execute_workflow", execute_attempt)
    monkeypatch.setattr(
        database.workflow_runs, "get_blocking_sequential_workflow_run", AsyncMock(return_value=run if blocked else None)
    )
    monkeypatch.setattr(
        service_module,
        "asyncio",
        ScopedAsyncio(timeout=record_admission_timeout, sleep=expire_timeout_at_gate_sleep),
    )

    result = await service.execute_workflow_with_retries(
        run.workflow_run_id, api_key=None, organization=SimpleNamespace(organization_id="o_test")
    )

    assert len(admission_budgets) == 1
    assert 55 < admission_budgets[0] <= 60
    attempts = await database.workflow_run_attempts.get_attempts(run.workflow_run_id)
    assert attempts[0].status == "timed_out"
    assert attempts[0].retry_decision == RETRY_DECISION_RETRY
    if blocked:
        assert gate_sleeps
        assert executed_attempts == [1]
        assert result.status == WorkflowRunStatus.timed_out
        assert attempts[1].started_at is None
        assert attempts[1].decision_reason == "sequential_gate_stopped"
    else:
        assert not gate_sleeps
        assert executed_attempts == [1, 2]
        assert result.status == WorkflowRunStatus.completed
        assert attempts[1].started_at is not None


@pytest.mark.asyncio
async def test_prepared_retry_recovery_keeps_its_admission_deadline(
    monkeypatch: pytest.MonkeyPatch,
    retry_gate_setup: tuple[AgentDB, WorkflowService, WorkflowRun],
) -> None:
    database, service, run = retry_gate_setup
    admission_deadlines: list[datetime] = []
    admission_budgets: list[float] = []
    elapsed_timeout = service_module._get_workflow_run_max_elapsed_timeout_seconds

    class WorkerLost(BaseException):
        pass

    def record_budget(gate_run: WorkflowRun) -> float:
        assert gate_run.started_at is not None
        admission_deadlines.append(gate_run.started_at.replace(tzinfo=UTC) + timedelta(minutes=1))
        remaining = elapsed_timeout(gate_run)
        admission_budgets.append(remaining)
        return remaining

    async def execute_attempt(**kwargs: Any) -> WorkflowRun:
        return await database.workflow_runs.update_workflow_run(run.workflow_run_id, status=WorkflowRunStatus.timed_out)

    execute = AsyncMock(side_effect=execute_attempt)
    monkeypatch.setattr(service, "execute_workflow", execute)
    monkeypatch.setattr(service_module, "_get_workflow_run_max_elapsed_timeout_seconds", record_budget)
    monkeypatch.setattr(
        database.workflow_runs, "get_blocking_sequential_workflow_run", AsyncMock(side_effect=WorkerLost)
    )

    with pytest.raises(WorkerLost):
        await service.execute_workflow_with_retries(
            run.workflow_run_id, api_key=None, organization=SimpleNamespace(organization_id="o_test")
        )

    attempts = await database.workflow_run_attempts.get_attempts(run.workflow_run_id)
    prepared_at = attempts[1].created_at.replace(tzinfo=UTC)
    assert attempts[1].started_at is None
    assert 55 < admission_budgets[0] <= 60
    current_time = prepared_at + timedelta(seconds=30)

    class RecoveryClock(datetime):
        @classmethod
        def now(cls, tz: tzinfo | None = None) -> datetime:
            return current_time.astimezone(tz) if tz is not None else current_time.replace(tzinfo=None)

    monkeypatch.setattr(service_module, "datetime", RecoveryClock)
    with pytest.raises(WorkerLost):
        await service.execute_workflow_with_retries(
            run.workflow_run_id, api_key=None, organization=SimpleNamespace(organization_id="o_test"), attempt_number=2
        )

    assert admission_budgets[1] == 30
    current_time = prepared_at + timedelta(minutes=1)
    result = await service.execute_workflow_with_retries(
        run.workflow_run_id, api_key=None, organization=SimpleNamespace(organization_id="o_test"), attempt_number=2
    )

    assert admission_deadlines == [prepared_at + timedelta(minutes=1)] * 3
    assert admission_budgets[2] == 0
    assert result.status == WorkflowRunStatus.timed_out
    assert [call.kwargs["attempt_number"] for call in execute.await_args_list] == [1]
    attempts = await database.workflow_run_attempts.get_attempts(run.workflow_run_id)
    assert attempts[1].started_at is None
    assert attempts[1].decision_reason == "sequential_gate_stopped"


@pytest.mark.asyncio
@pytest.mark.parametrize("escaped_exception", [False, True])
async def test_forced_session_is_pinned_for_first_and_prepared_retry_attempt(
    monkeypatch: pytest.MonkeyPatch,
    forced_session_setup: tuple[AgentDB, WorkflowService, Workflow],
    escaped_exception: bool,
) -> None:
    database, service, workflow = forced_session_setup

    run = await service.create_workflow_run(
        workflow_request=WorkflowRequestBody(),
        workflow_permanent_id="wpid_test",
        workflow_id="wf_1",
        organization_id="o_test",
        workflow=workflow,
    )

    assert run.browser_session_id == "pbs_forced"
    attempts = await database.workflow_run_attempts.get_attempts(run.workflow_run_id)
    assert len(attempts) == 1
    assert attempts[0].attempt_number == 1
    assert attempts[0].pinned_browser_session_id == "pbs_forced"

    context = skyvern_context.SkyvernContext(workflow_run_id=run.workflow_run_id, organization_id="o_test")
    monkeypatch.setattr(skyvern_context, "current", lambda: context)
    monkeypatch.setattr(skyvern_context, "ensure_context", lambda: context)
    monkeypatch.setattr(app, "BROWSER_MANAGER", RealBrowserManager())
    generations: list[str] = []
    released_generations: list[str] = []
    browser_accesses: list[tuple[int, str | None]] = []
    owned_generation: str | None = None

    async def begin_session(**kwargs: Any) -> str:
        nonlocal owned_generation
        assert kwargs["browser_session_id"] == "pbs_forced"
        assert kwargs["runnable_id"] == run.workflow_run_id
        assert owned_generation is None
        owned_generation = f"lease_{len(generations) + 1}"
        generations.append(owned_generation)
        return owned_generation

    async def release_session(**kwargs: Any) -> bool:
        nonlocal owned_generation
        assert kwargs["expected_runnable_id"] == run.workflow_run_id
        assert kwargs["expected_runnable_generation_id"] == owned_generation
        assert owned_generation is not None
        released_generations.append(owned_generation)
        owned_generation = None
        return True

    async def execute_attempt(*, attempt_number: int, browser_session_id: str | None, **kwargs: Any) -> WorkflowRun:
        kwargs["on_execution_authority"](attempt_number, kwargs["dispatch_claim_started_at"])
        current_run = await database.workflow_runs.get_workflow_run(run.workflow_run_id, "o_test")
        assert current_run is not None
        session_id = browser_session_id or current_run.browser_session_id
        assert session_id == "pbs_forced"
        if attempt_number > 1:
            assert context.browser_session_id == session_id
        await service._ensure_browser_session_lease(
            organization_id="o_test", workflow_run_id=run.workflow_run_id, browser_session_id=session_id
        )
        generation = context.browser_session_runnable_generation_id
        browser_accesses.append((attempt_number, generation))
        owns_session = (
            context.browser_session_runnable_id == run.workflow_run_id
            and generation == owned_generation
            and generation not in released_generations
        )
        if owns_session:
            await service._clean_up_workflow_browser(
                workflow_run=current_run, close_browser_on_completion=False, browser_session_id=session_id
            )
        status = WorkflowRunStatus.completed if attempt_number == 2 and owns_session else WorkflowRunStatus.failed
        current_run = await database.workflow_runs.update_workflow_run(run.workflow_run_id, status=status)
        async with database.Session() as session:
            attempt = await session.scalar(
                select(WorkflowRunAttemptModel).where(
                    WorkflowRunAttemptModel.workflow_run_id == run.workflow_run_id,
                    WorkflowRunAttemptModel.attempt_number == attempt_number,
                )
            )
            assert attempt is not None
            attempt.status = status.value
            attempt.retry_decision = RETRY_DECISION_RETRY if attempt_number == 1 else RETRY_DECISION_FINAL
            await session.commit()
        if attempt_number == 1 and escaped_exception:
            raise RuntimeError("escaped after browser cleanup")
        return current_run

    monkeypatch.setattr(app.PERSISTENT_SESSIONS_MANAGER, "begin_session", begin_session)
    monkeypatch.setattr(app.PERSISTENT_SESSIONS_MANAGER, "release_browser_session", release_session)
    monkeypatch.setattr(service, "execute_workflow", execute_attempt)
    monkeypatch.setattr(service, "_wait_for_retry_sequential_clearance", AsyncMock(return_value=None))
    monkeypatch.setattr(service, "_run_interim_side_effects_with_retries", AsyncMock(return_value="released"))
    monkeypatch.setattr(service, "_run_terminal_side_effects_with_retries", AsyncMock(return_value="released"))

    result = await service.execute_workflow_with_retries(
        run.workflow_run_id, api_key=None, organization=SimpleNamespace(organization_id="o_test")
    )

    assert browser_accesses == [(1, "lease_1"), (2, "lease_2")]
    assert generations == released_generations == ["lease_1", "lease_2"]
    assert result.status == WorkflowRunStatus.completed
    assert context.browser_session_id == "pbs_forced"
    attempts = await database.workflow_run_attempts.get_attempts(run.workflow_run_id)
    assert [(attempt.attempt_number, attempt.pinned_browser_session_id) for attempt in attempts] == [
        (1, "pbs_forced"),
        (2, "pbs_forced"),
    ]
    prepared_run = await database.workflow_runs.get_workflow_run(run.workflow_run_id, "o_test")
    assert prepared_run is not None
    assert prepared_run.browser_session_id == "pbs_forced"


@pytest.mark.asyncio
@pytest.mark.parametrize("session_source", ["forced", "explicit", "unpinned"])
async def test_retry_preparation_preserves_session_after_transient_forced_pin_failure(
    monkeypatch: pytest.MonkeyPatch,
    forced_session_setup: tuple[AgentDB, WorkflowService, Workflow],
    session_source: str,
) -> None:
    database, service, workflow = forced_session_setup
    pin_session = database.workflow_run_attempts.pin_first_attempt_browser_session_if_unset
    pin_calls = 0

    async def fail_first_pin(**kwargs: Any) -> None:
        nonlocal pin_calls
        pin_calls += 1
        if pin_calls == 1:
            raise RuntimeError("pin failed")
        await pin_session(**kwargs)

    monkeypatch.setattr(database.workflow_run_attempts, "pin_first_attempt_browser_session_if_unset", fail_first_pin)
    if session_source != "forced":
        monkeypatch.setattr(app.EXPERIMENTATION_PROVIDER, "is_feature_enabled_cached", AsyncMock(return_value=False))
    if session_source == "explicit":
        monkeypatch.setattr(
            database.browser_sessions,
            "get_persistent_browser_session",
            AsyncMock(
                return_value=SimpleNamespace(
                    browser_profile_id=None,
                    browser_vendor=None,
                    runnable_id="wr_other",
                    status="running",
                    completed_at=None,
                    close_requested_at=None,
                )
            ),
        )
    run = await service.create_workflow_run(
        workflow_request=WorkflowRequestBody(
            browser_session_id="pbs_explicit" if session_source == "explicit" else None
        ),
        workflow_permanent_id="wpid_test",
        workflow_id="wf_1",
        organization_id="o_test",
        workflow=workflow,
    )
    await database.workflow_runs.update_workflow_run(
        run.workflow_run_id,
        status=WorkflowRunStatus.failed,
        browser_session_id="pbs_leased" if session_source == "unpinned" else run.browser_session_id,
    )
    async with database.Session() as session:
        attempt = await session.get(WorkflowRunAttemptModel, (run.workflow_run_id, 1))
        assert attempt is not None
        attempt.status = WorkflowRunStatus.failed.value
        attempt.retry_decision = RETRY_DECISION_RETRY
        await session.commit()

    preparation = await service_module.prepare_next_attempt_result(run.workflow_run_id, "o_test", 1)

    expected_session_id = {"forced": "pbs_forced", "explicit": "pbs_explicit", "unpinned": None}[session_source]
    assert preparation.status == "inserted"
    assert preparation.pinned_browser_session_id == expected_session_id
    attempts = await database.workflow_run_attempts.get_attempts(run.workflow_run_id)
    assert [attempt.pinned_browser_session_id for attempt in attempts] == [expected_session_id, expected_session_id]
    reopened = await database.workflow_runs.get_workflow_run(run.workflow_run_id, "o_test")
    assert reopened is not None
    assert reopened.browser_session_id == expected_session_id
    assert pin_calls == (2 if session_source == "forced" else 0)


@pytest.mark.asyncio
@pytest.mark.parametrize("entry_point", ["workflow_run", "task"])
async def test_submission_refuses_a_session_past_its_lifetime_and_accepts_a_live_one(
    monkeypatch: pytest.MonkeyPatch,
    forced_session_setup: tuple[AgentDB, WorkflowService, Workflow],
    entry_point: str,
) -> None:
    database, service, workflow = forced_session_setup
    monkeypatch.setattr(app.EXPERIMENTATION_PROVIDER, "is_feature_enabled_cached", AsyncMock(return_value=False))
    ended_at = datetime(2026, 9, 28, 17, 1)
    async with database.Session() as session:
        session.add_all(
            [
                PersistentBrowserSessionModel(
                    persistent_browser_session_id="pbs_expired",
                    organization_id="o_test",
                    status="timeout",
                    timeout_minutes=240,
                    started_at=ended_at - timedelta(minutes=240),
                    completed_at=ended_at,
                    created_by="api",
                ),
                PersistentBrowserSessionModel(
                    persistent_browser_session_id="pbs_live",
                    organization_id="o_test",
                    status="running",
                    timeout_minutes=240,
                    started_at=datetime.now(UTC).replace(tzinfo=None),
                    created_by="api",
                ),
                PersistentBrowserSessionModel(
                    persistent_browser_session_id="pbs_external",
                    organization_id="o_test",
                    status="running",
                    timeout_minutes=240,
                    started_at=datetime.now(UTC).replace(tzinfo=None),
                    browser_vendor=EXTERNAL_CDP_BROWSER_VENDOR,
                    upstream_cdp_url="sealed-address",
                ),
            ]
        )
        await session.commit()

    async def submit(browser_session_id: str) -> str | None:
        if entry_point == "task":
            task = await ForgeAgent().create_task(
                TaskRequest(url="https://example.com", browser_session_id=browser_session_id), "o_test"
            )
            return task.browser_session_id
        run = await service.create_workflow_run(
            workflow_request=WorkflowRequestBody(browser_session_id=browser_session_id),
            workflow_permanent_id="wpid_test",
            workflow_id="wf_1",
            organization_id="o_test",
            workflow=workflow,
        )
        return run.browser_session_id

    with pytest.raises(BrowserSessionExpired) as refused:
        await submit("pbs_expired")

    assert refused.value.status_code == 410
    assert "pbs_expired" in refused.value.message
    async with database.Session() as session:
        assert (await session.scalars(select(WorkflowRunModel))).all() == []
        assert (await session.scalars(select(TaskModel))).all() == []
    assert await submit("pbs_live") == "pbs_live"
    assert await submit("pbs_external") == "pbs_external"


@pytest.mark.asyncio
@pytest.mark.parametrize("failure_point", ["assignment", "pin"])
async def test_forced_session_assignment_precedes_required_pin(
    monkeypatch: pytest.MonkeyPatch,
    forced_session_setup: tuple[AgentDB, WorkflowService, Workflow],
    failure_point: str,
) -> None:
    database, service, workflow = forced_session_setup
    close_session = AsyncMock()
    monkeypatch.setattr(app.PERSISTENT_SESSIONS_MANAGER, "close_session", close_session)
    if failure_point == "assignment":
        monkeypatch.setattr(
            database.workflow_runs, "update_workflow_run", AsyncMock(side_effect=RuntimeError("assignment failed"))
        )
    else:
        monkeypatch.setattr(
            database.workflow_run_attempts,
            "pin_first_attempt_browser_session_if_unset",
            AsyncMock(side_effect=RuntimeError("pin failed")),
        )

    if failure_point == "pin":
        with pytest.raises(RuntimeError, match="pin failed"):
            await service.create_workflow_run(
                workflow_request=WorkflowRequestBody(),
                workflow_permanent_id="wpid_test",
                workflow_id="wf_1",
                workflow_run_id="wr_pin_failure",
                organization_id="o_test",
                workflow=workflow,
            )
        run = await database.workflow_runs.get_workflow_run("wr_pin_failure", "o_test")
        assert run is not None
        assert run.status == WorkflowRunStatus.failed
        assert run.finished_at is not None
        assert run.failure_reason == "Failed to pin forced browser session for workflow retry"
        close_session.assert_awaited_once_with("o_test", "pbs_forced", reason=BrowserSessionCloseReason.aborted)
        assert database.workflow_run_attempts.pin_first_attempt_browser_session_if_unset.await_count == 2
    else:
        run = await service.create_workflow_run(
            workflow_request=WorkflowRequestBody(),
            workflow_permanent_id="wpid_test",
            workflow_id="wf_1",
            organization_id="o_test",
            workflow=workflow,
        )

    persisted_run = await database.workflow_runs.get_workflow_run(run.workflow_run_id, "o_test")
    assert persisted_run is not None
    expected_session_id = None if failure_point == "assignment" else "pbs_forced"
    assert run.browser_session_id == persisted_run.browser_session_id == expected_session_id
    attempts = await database.workflow_run_attempts.get_attempts(run.workflow_run_id)
    assert len(attempts) == 1
    assert attempts[0].pinned_browser_session_id is None
    if failure_point == "pin":
        assert attempts[0].status == WorkflowRunStatus.failed.value
        assert attempts[0].retry_decision == "abandoned"


@pytest.mark.asyncio
@pytest.mark.parametrize("deleted_version", [False, True])
@pytest.mark.parametrize("seed_source", [BrowserSeedSource.credential, BrowserSeedSource.own_memory])
async def test_retry_execution_resolves_rotated_seed_before_browser_creation(
    monkeypatch: pytest.MonkeyPatch,
    sqlite_engine: AsyncEngine,
    deleted_version: bool,
    seed_source: BrowserSeedSource,
) -> None:
    database = AgentDB("sqlite+aiosqlite:///:memory:", db_engine=sqlite_engine)
    now = datetime.now(UTC)
    credential_parameter = {
        "parameter_type": "credential",
        "key": "login",
        "credential_parameter_id": "cp_login",
        "workflow_id": "wf_1",
        "credential_id": "cred_a",
        "credential_ids": ["cred_a", "cred_b"],
        "created_at": now.isoformat(),
        "modified_at": now.isoformat(),
    }
    own_memory = seed_source == BrowserSeedSource.own_memory
    async with database.Session() as session:
        session.add_all(
            [
                WorkflowModel(
                    workflow_id="wf_1",
                    workflow_permanent_id="wpid_test",
                    organization_id="o_test",
                    title="Workflow",
                    persist_browser_session=own_memory,
                    browser_profile_key="{{ login }}" if own_memory else None,
                    workflow_definition={
                        "parameters": [credential_parameter],
                        "blocks": [
                            {
                                "block_type": "login",
                                "label": "login",
                                "parameters": [credential_parameter],
                                "output_parameter": {
                                    "parameter_type": "output",
                                    "key": "login_output",
                                    "output_parameter_id": "op_login",
                                    "workflow_id": "wf_1",
                                    "created_at": now.isoformat(),
                                    "modified_at": now.isoformat(),
                                },
                            }
                        ],
                        "retry_policy": {"max_retries": 1, "retry_on": [{"status": "failed"}]},
                    },
                ),
                WorkflowRunModel(
                    workflow_run_id="wr_test",
                    workflow_id="wf_1",
                    workflow_permanent_id="wpid_test",
                    organization_id="o_test",
                    status="failed",
                    browser_profile_id="bp_a",
                    browser_seed_source=seed_source,
                    browser_sink_profile_id="bp_a" if own_memory else None,
                    max_elapsed_time_minutes=1,
                ),
                WorkflowRunAttemptModel(
                    workflow_run_id="wr_test",
                    organization_id="o_test",
                    attempt_number=1,
                    status="failed",
                    retry_decision="retry",
                ),
                WorkflowRunCredentialSelectionModel(
                    workflow_run_id="wr_test",
                    organization_id="o_test",
                    workflow_permanent_id="wpid_test",
                    parameter_key="login",
                    credential_id="cred_a",
                ),
                *[
                    CredentialModel(
                        credential_id=f"cred_{suffix}",
                        organization_id="o_test",
                        name=f"Credential {suffix}",
                        credential_type="password",
                        item_id=f"item_{suffix}",
                        run_sequentially=False,
                        browser_profile_id=None if own_memory else f"bp_{suffix}",
                    )
                    for suffix in ("a", "b")
                ],
                *[
                    BrowserProfileModel(
                        browser_profile_id=f"bp_{suffix}",
                        organization_id="o_test",
                        name=f"Profile {suffix}",
                        is_managed=own_memory,
                        workflow_permanent_id="wpid_test" if own_memory else None,
                        browser_profile_key_digest=(
                            build_browser_profile_key_digest(f"cred_{suffix}") if own_memory else None
                        ),
                    )
                    for suffix in ("a", "b")
                ],
            ]
        )
        await session.commit()
    workflow = await database.workflows.get_workflow("wf_1", organization_id="o_test")
    assert workflow is not None
    if deleted_version:
        async with database.Session() as session:
            row = await session.get(WorkflowModel, "wf_1")
            assert row is not None
            row.deleted_at = now
            await session.commit()
        assert await database.workflows.get_workflow("wf_1", organization_id="o_test") is None

    svc = WorkflowService()
    _patch_execute_workflow_deps(monkeypatch, svc, workflow, _execute_workflow_run(WorkflowRunStatus.completed))
    monkeypatch.setattr(app, "DATABASE", database)
    monkeypatch.setattr(svc, "get_workflow_run", WorkflowService.get_workflow_run.__get__(svc))
    monkeypatch.setattr(svc, "get_workflow", WorkflowService.get_workflow.__get__(svc))
    monkeypatch.setattr(
        svc, "get_workflow_by_workflow_run_id", WorkflowService.get_workflow_by_workflow_run_id.__get__(svc)
    )
    monkeypatch.setattr(app.AGENT_FUNCTION, "is_browser_memory_engine_enabled", AsyncMock(return_value=True))
    monkeypatch.setattr(svc, "_managed_browser_profile_has_content", AsyncMock(return_value=True))
    monkeypatch.setattr(svc, "_reconcile_managed_browser_profile_proxy_pin", AsyncMock())
    monkeypatch.setattr(svc, "_maybe_pin_credential_profile_ip", AsyncMock(side_effect=lambda **kw: kw["workflow_run"]))
    monkeypatch.setattr(svc, "clean_up_workflow", AsyncMock())
    monkeypatch.setattr(svc, "_clean_up_workflow_browser", AsyncMock(return_value=_browser_cleanup_result()))
    monkeypatch.setattr(svc, "_finalize_workflow_run_status", AsyncMock(side_effect=lambda **kw: kw["workflow_run"]))
    monkeypatch.setattr(svc, "_run_terminal_side_effects_with_retries", AsyncMock())
    monkeypatch.setattr(
        service_module, "get_recorded_decision", AsyncMock(return_value=RetryDecision(False, 2, 0, True, "no_match"))
    )

    async def mark_running(workflow_run_id: str) -> WorkflowRun:
        return await database.workflow_runs.update_workflow_run(
            workflow_run_id, status=WorkflowRunStatus.running, started_at=datetime.now(UTC).replace(tzinfo=None)
        )

    monkeypatch.setattr(svc, "mark_workflow_run_as_running", mark_running)
    browser_seeds: list[str | None] = []

    async def execute_blocks(**kwargs: Any) -> tuple[WorkflowRun, set[str]]:
        browser_seeds.append(kwargs["browser_profile_id"])
        return await database.workflow_runs.update_workflow_run("wr_test", status=WorkflowRunStatus.completed), set()

    monkeypatch.setattr(svc, "_execute_workflow_blocks", execute_blocks)
    preparation = await database.workflow_runs.prepare_next_attempt_atomic(
        workflow_run_id="wr_test",
        organization_id="o_test",
        from_attempt=1,
        expected_status=WorkflowRunStatus.failed,
        browser_session_id=None,
    )
    assert preparation.status == "inserted"

    result = await svc.execute_workflow_with_retries(
        workflow_run_id="wr_test",
        api_key=None,
        organization=SimpleNamespace(organization_id="o_test"),
        attempt_number=2,
    )

    assert result.status == WorkflowRunStatus.completed
    assert browser_seeds == ["bp_b"]
    assert result.browser_profile_id == "bp_b"
    assert result.browser_seed_source == seed_source
    assert result.browser_sink_profile_id == ("bp_b" if own_memory else None)
    assert await database.workflow_run_credential_selections.get_selection("wr_test", "login") == "cred_b"


@pytest.mark.asyncio
async def test_run_admission_failure_finishes_without_browser_or_blocks(monkeypatch: pytest.MonkeyPatch) -> None:
    svc = WorkflowService()
    failed_run = _execute_workflow_run(WorkflowRunStatus.failed)
    _patch_execute_workflow_deps(monkeypatch, svc, _execute_workflow(), failed_run)
    monkeypatch.setattr(
        app.AGENT_FUNCTION, "before_workflow_run_start", AsyncMock(side_effect=RuntimeError("admission denied"))
    )
    monkeypatch.setattr(svc, "_mark_workflow_run_as_failed_for_dispatch", AsyncMock(return_value=failed_run))
    cleanup = AsyncMock()
    monkeypatch.setattr(svc, "clean_up_workflow", cleanup)

    result = await _run_execute_workflow(svc)

    assert result.status == WorkflowRunStatus.failed
    svc.mark_workflow_run_as_running.assert_not_awaited()
    svc.auto_create_browser_session_if_needed.assert_not_awaited()
    svc._execute_workflow_blocks.assert_not_awaited()
    cleanup.assert_awaited_once()


@pytest.mark.asyncio
async def test_superseded_admission_does_not_finalize_replacement(monkeypatch: pytest.MonkeyPatch) -> None:
    svc = WorkflowService()
    _patch_execute_workflow_deps(
        monkeypatch, svc, _execute_workflow(), _execute_workflow_run(WorkflowRunStatus.running)
    )
    monkeypatch.setattr(
        app.AGENT_FUNCTION,
        "before_workflow_run_start",
        AsyncMock(side_effect=service_module.WorkflowAttemptDispatchSuperseded("wr_test", 1)),
    )
    finalize = AsyncMock()
    monkeypatch.setattr(svc, "mark_workflow_run_as_failed_if_not_final", finalize)
    with pytest.raises(service_module.WorkflowAttemptDispatchSuperseded):
        await _run_execute_workflow(svc)
    finalize.assert_not_awaited()
    svc.mark_workflow_run_as_running.assert_not_awaited()
    svc._execute_workflow_blocks.assert_not_awaited()


@pytest.mark.asyncio
async def test_forced_browser_deferral_preserves_preparation_and_first_attempt_pin(
    monkeypatch: pytest.MonkeyPatch,
    forced_session_setup: tuple[AgentDB, WorkflowService, Workflow],
) -> None:
    database, svc, workflow = forced_session_setup
    monkeypatch.setattr(app.AGENT_FUNCTION, "should_defer_workflow_browser_creation", AsyncMock(return_value=True))
    request = WorkflowRequestBody(data={})
    run = await svc.create_workflow_run(
        workflow_request=request,
        workflow_permanent_id=workflow.workflow_permanent_id,
        workflow_id=workflow.workflow_id,
        organization_id="o_test",
        workflow=workflow,
    )
    assert run.browser_session_id is None
    app.PERSISTENT_SESSIONS_MANAGER.create_session.assert_not_awaited()
    attempts = await database.workflow_run_attempts.get_attempts(run.workflow_run_id)
    assert attempts[0].pinned_browser_session_id is None

    run = await svc._prepare_forced_browser_session(
        workflow=workflow, workflow_run=run, workflow_request=request, has_attempt_row=True
    )
    assert run.browser_session_id == "pbs_forced"
    attempts = await database.workflow_run_attempts.get_attempts(run.workflow_run_id)
    assert attempts[0].pinned_browser_session_id == "pbs_forced"
    assert app.PERSISTENT_SESSIONS_MANAGER.create_session.await_args.kwargs["workflow_run_id"] == run.workflow_run_id


@pytest.mark.asyncio
@pytest.mark.parametrize(
    "owner", ["replaced", "next_attempt", "current", "no_policy", "canceled", "finalized", "write_error"]
)
async def test_admission_error_respects_persisted_dispatch_owner(
    monkeypatch: pytest.MonkeyPatch,
    forced_session_setup: tuple[AgentDB, WorkflowService, Workflow],
    owner: str,
) -> None:
    database, svc, workflow = forced_session_setup
    _patch_execute_workflow_deps(
        monkeypatch,
        svc,
        _execute_workflow(retry_policy=None if owner == "no_policy" else workflow.workflow_definition.retry_policy),
        _execute_workflow_run(WorkflowRunStatus.failed),
    )
    monkeypatch.setattr(app, "DATABASE", database)
    monkeypatch.setattr(app, "WORKFLOW_SERVICE", svc)
    monkeypatch.setattr(svc, "get_workflow_run", WorkflowService.get_workflow_run.__get__(svc))
    monkeypatch.setattr(svc, "clean_up_workflow", AsyncMock())
    monkeypatch.setattr(app.AGENT_FUNCTION, "record_run_duration", AsyncMock())
    monkeypatch.setattr(svc, "_apply_completion_run_tags_best_effort", AsyncMock())
    monkeypatch.setattr(svc, "_schedule_workflow_run_terminal_hooks", Mock())
    save_logs = AsyncMock()
    monkeypatch.setattr(workflow_runs_repository, "save_workflow_run_logs", save_logs)
    async with database.Session() as session:
        session.add(
            WorkflowRunModel(
                workflow_run_id="wr_test",
                workflow_id="wf_1",
                workflow_permanent_id="wpid_test",
                organization_id="o_test",
                status="queued",
            )
        )
        session.add(
            TaskRunModel(
                run_id="wr_test",
                organization_id="o_test",
                task_run_type="workflow",
                status="queued",
            )
        )
        await session.commit()
    claim = None
    if owner != "no_policy":
        await database.workflow_run_attempts.create_attempt("wr_test", "o_test", 1, "queued")
        claim = await database.workflow_run_attempts.claim_prepared_attempt_execution("wr_test", "o_test", 1)
        assert claim is not None
    replacement_claim = None

    async def fail_admission(*args: Any, **kwargs: Any) -> None:
        nonlocal replacement_claim
        if owner == "replaced":
            assert await database.workflow_run_attempts.release_stale_dispatch_claim(
                "wr_test", "o_test", 1, stale_before=datetime.now(UTC) + timedelta(minutes=1)
            )
            replacement_claim = await database.workflow_run_attempts.claim_prepared_attempt_execution(
                "wr_test", "o_test", 1
            )
            assert replacement_claim is not None and replacement_claim != claim
        elif owner == "next_attempt":
            await database.workflow_run_attempts.create_attempt("wr_test", "o_test", 2, "queued")
        elif owner == "canceled":
            await database.workflow_runs.update_workflow_run_if_not_final("wr_test", WorkflowRunStatus.canceled)
        elif owner == "finalized":
            await database.workflow_run_attempts.finalize_attempt(
                "wr_test",
                1,
                status="failed",
                failure_reason="previous owner finished",
                failure_category=None,
                error_codes=[],
                finished_at=datetime.now(UTC),
                retry_decision=RETRY_DECISION_FINAL,
                decision_reason="no_match",
                next_attempt_at=None,
            )
        elif owner == "write_error":
            monkeypatch.setattr(
                database.workflow_runs,
                "finalize_workflow_run_for_dispatch",
                AsyncMock(side_effect=RuntimeError("finalization unavailable")),
            )
        raise RuntimeError("admission database unavailable")

    monkeypatch.setattr(app.AGENT_FUNCTION, "before_workflow_run_start", fail_admission)
    save_logs.reset_mock()
    if owner in {"replaced", "next_attempt", "finalized"}:
        with pytest.raises(service_module.WorkflowAttemptDispatchSuperseded):
            await _run_execute_workflow(svc, dispatch_claim_started_at=claim)
    elif owner == "write_error":
        with pytest.raises(RuntimeError, match="finalization unavailable"):
            await _run_execute_workflow(svc, dispatch_claim_started_at=claim)
    else:
        await _run_execute_workflow(svc, dispatch_claim_started_at=claim)
    if svc._background_tasks:
        await asyncio.gather(*list(svc._background_tasks))
    persisted = await database.workflow_runs.get_workflow_run("wr_test", "o_test")
    assert persisted is not None
    attempts = await database.workflow_run_attempts.get_attempts("wr_test")
    won = owner in {"current", "no_policy"}
    assert persisted.status == (
        WorkflowRunStatus.failed
        if won
        else WorkflowRunStatus.canceled
        if owner == "canceled"
        else WorkflowRunStatus.queued
    )
    if won:
        assert persisted.finished_at is not None and persisted.failure_reason
        svc.clean_up_workflow.assert_awaited_once()
        save_logs.assert_awaited_once_with("wr_test")
        app.AGENT_FUNCTION.record_run_duration.assert_awaited_once()
        if attempts:
            assert attempts[0].retry_decision == RETRY_DECISION_RETRY
            assert attempts[0].finished_at is not None
        async with database.Session() as session:
            mirror = await session.scalar(select(TaskRunModel).where(TaskRunModel.run_id == "wr_test"))
            assert mirror is not None and mirror.status == "failed"
            assert mirror.finished_at == persisted.finished_at.replace(tzinfo=None)
    else:
        svc.clean_up_workflow.assert_not_awaited()
        app.AGENT_FUNCTION.record_run_duration.assert_not_awaited()
        if owner != "canceled":
            save_logs.assert_not_awaited()
        if owner == "replaced":
            assert attempts[0].started_at == replacement_claim
            assert attempts[0].retry_decision is None
            assert attempts[0].finished_at is None
    svc.mark_workflow_run_as_running.assert_not_awaited()
    svc.auto_create_browser_session_if_needed.assert_not_awaited()
    svc._execute_workflow_blocks.assert_not_awaited()


@pytest.mark.asyncio
@pytest.mark.parametrize("pin_ip", [False, True])
@pytest.mark.parametrize(
    "seed_source,start_fresh",
    [
        (BrowserSeedSource.override, False),
        (BrowserSeedSource.override, True),
        (BrowserSeedSource.own_memory, False),
        (BrowserSeedSource.credential, False),
        (BrowserSeedSource.picked, False),
        (BrowserSeedSource.fresh, True),
        (BrowserSeedSource.fresh, False),
    ],
)
async def test_deferred_execution_uses_persisted_seed_for_forced_browser(
    monkeypatch: pytest.MonkeyPatch,
    forced_session_setup: tuple[AgentDB, WorkflowService, Workflow],
    seed_source: BrowserSeedSource,
    start_fresh: bool,
    pin_ip: bool,
) -> None:
    database, svc, workflow = forced_session_setup
    monkeypatch.setattr(app.AGENT_FUNCTION, "should_defer_workflow_browser_creation", AsyncMock(return_value=True))
    profile_id = None if seed_source == BrowserSeedSource.fresh else "bp_seed"
    workflow.persist_browser_session = True
    workflow.pin_saved_session_ip = pin_ip
    proxy = service_module.ProxyLocation.RESIDENTIAL_ISP
    async with database.Session() as session:
        session.add(
            BrowserProfileModel(
                browser_profile_id="bp_seed",
                organization_id="o_test",
                name="Saved seed",
                is_managed=True,
            )
        )
        await session.commit()
    run = await svc.create_workflow_run(
        workflow_request=WorkflowRequestBody(
            start_fresh_browser=start_fresh,
            proxy_location=proxy,
            browser_type="msedge",
        ),
        workflow_permanent_id="wpid_test",
        workflow_id="wf_1",
        workflow_run_id="wr_test",
        organization_id="o_test",
        workflow=workflow,
    )
    assert run.browser_session_id is None
    app.PERSISTENT_SESSIONS_MANAGER.create_session.assert_not_awaited()
    # Setup has resolved the seed before the executing worker reconstructs its request.
    run = await database.workflow_runs.update_workflow_run(
        "wr_test",
        browser_profile_id=profile_id,
        browser_seed_source=seed_source,
        browser_sink_profile_id="bp_seed" if seed_source == BrowserSeedSource.own_memory else None,
    )
    execution_workflow = _execute_workflow()
    execution_workflow.pin_saved_session_ip = pin_ip
    execution_workflow.proxy_location = proxy
    execution_workflow.browser_profile_key = None
    _patch_execute_workflow_deps(monkeypatch, svc, execution_workflow, run)
    monkeypatch.setattr(app, "DATABASE", database)
    monkeypatch.setattr(svc, "get_workflow_run", WorkflowService.get_workflow_run.__get__(svc))
    monkeypatch.setattr(svc, "mark_workflow_run_as_running", WorkflowService.mark_workflow_run_as_running.__get__(svc))
    monkeypatch.setattr(
        svc,
        "_resolve_managed_browser_profile_for_run_request",
        WorkflowService._resolve_managed_browser_profile_for_run_request.__get__(svc),
    )
    monkeypatch.setattr(svc, "_seed_managed_browser_profile_from_legacy_session", AsyncMock())
    monkeypatch.setattr(svc, "_reconcile_managed_browser_profile_proxy_pin", AsyncMock())
    monkeypatch.setattr(app.AGENT_FUNCTION, "should_defer_workflow_browser_creation", AsyncMock(return_value=True))
    monkeypatch.setattr(
        app.WORKFLOW_CONTEXT_MANAGER,
        "initialize_workflow_run_context",
        AsyncMock(side_effect=BaseException("stop after preparation")),
    )
    with pytest.raises(BaseException, match="stop after preparation"):
        await _run_execute_workflow(svc)
    if svc._background_tasks:
        await asyncio.gather(*list(svc._background_tasks))
    persisted = await database.workflow_runs.get_workflow_run("wr_test", "o_test")
    assert persisted is not None and persisted.browser_profile_id == profile_id
    assert persisted.browser_seed_source == seed_source
    assert persisted.reuse_bound_key == service_module.REUSE_ADMISSION_OFF_DISABLED
    attempts = await database.workflow_run_attempts.get_attempts("wr_test")
    create_session = app.PERSISTENT_SESSIONS_MANAGER.create_session
    if pin_ip and profile_id is None:
        create_session.assert_not_awaited()
        assert persisted.browser_session_id is None and attempts[0].pinned_browser_session_id is None
    else:
        create_session.assert_awaited_once()
        creation = create_session.await_args.kwargs
        assert creation["browser_profile_id"] == profile_id
        assert creation["proxy_location"] == proxy
        assert creation["browser_type"].value == "msedge"
        assert creation["inherit_profile_proxy"] is True
        assert creation["workflow_run_id"] == "wr_test"
        assert persisted.browser_session_id == attempts[0].pinned_browser_session_id == "pbs_forced"


@pytest.mark.asyncio
@pytest.mark.parametrize(
    "case",
    [
        "on",
        "off",
        "fallback",
        "refused",
        "assignment",
        "replaced",
        "stale_fallback",
        "address",
        "reuse",
        "refresh_none",
        "refresh_supplied",
    ],
)
async def test_deferred_force_launch_decision_survives_publication_and_reentry(monkeypatch, forced_session_setup, case):
    database, svc, workflow = forced_session_setup
    selected = case != "off"
    monkeypatch.setattr(app.AGENT_FUNCTION, "should_defer_workflow_browser_creation", AsyncMock(return_value=True))
    flag = AsyncMock(side_effect=lambda name, *a, **k: selected and name == "FORCE_BROWSER_SESSION")
    monkeypatch.setattr(app.EXPERIMENTATION_PROVIDER, "is_feature_enabled_cached", flag)
    run = await svc.create_workflow_run(
        workflow_request=WorkflowRequestBody(
            start_fresh_browser=case not in {"reuse", "address"},
            reuse_browser_session=case != "address",
            browser_address="http://browser.invalid" if case == "address" else None,
        ),
        workflow_permanent_id="wpid_test",
        workflow_id="wf_1",
        workflow_run_id="wr_test",
        organization_id="o_test",
        workflow=workflow,
    )
    run, reuse = await svc.resolve_and_persist_reuse_bound_key(
        workflow=workflow, workflow_run=run, effective_reuse=case == "reuse"
    )
    assert reuse is (case == "reuse")
    assert "reuse_bound_key" not in run.model_dump()
    run = await database.workflow_runs.update_workflow_run("wr_test", browser_seed_source=BrowserSeedSource.fresh)
    assert await database.workflow_runs.queue_initial_dispatch("wr_test", 1)
    claim = await database.workflow_run_attempts.claim_prepared_attempt_execution("wr_test", "o_test", 1)
    assert claim is not None
    execution = _execute_workflow()
    execution.pin_saved_session_ip = False
    execution.proxy_location = None
    _patch_execute_workflow_deps(monkeypatch, svc, execution, run)
    monkeypatch.setattr(app, "DATABASE", database)
    monkeypatch.setattr(svc, "get_workflow_run", WorkflowService.get_workflow_run.__get__(svc))
    monkeypatch.setattr(svc, "mark_workflow_run_as_running", WorkflowService.mark_workflow_run_as_running.__get__(svc))
    monkeypatch.setattr(app.AGENT_FUNCTION, "should_defer_workflow_browser_creation", AsyncMock(return_value=True))
    flag.side_effect = lambda name, *a, **k: not selected and name == "FORCE_BROWSER_SESSION"
    flag.reset_mock()
    monkeypatch.setattr(svc, "clean_up_workflow", AsyncMock())
    monkeypatch.setattr(svc, "_apply_completion_run_tags_best_effort", AsyncMock())
    monkeypatch.setattr(svc, "_schedule_workflow_run_terminal_hooks", Mock())
    monkeypatch.setattr(workflow_runs_repository, "save_workflow_run_logs", AsyncMock())
    monkeypatch.setattr(app.AGENT_FUNCTION, "record_run_duration", AsyncMock())
    # Stop at actual application execution, after the ordinary browser entry point.
    svc._execute_workflow_blocks.side_effect = asyncio.CancelledError()
    _patch_browser_cleanup(monkeypatch, svc, [])
    _patch_finalize(monkeypatch, svc, [], run)

    if case.startswith("refresh_"):
        original = "pbs_original" if case == "refresh_supplied" else None

        async def observe_predecessor(workflow_run, **kwargs):
            workflow_run.browser_session_id = original
            await database.workflow_runs.update_workflow_run("wr_test", browser_session_id=original)

        monkeypatch.setattr(app.AGENT_FUNCTION, "before_workflow_run_start", observe_predecessor)
        mark_running = svc.mark_workflow_run_as_running

        async def refresh_after_competing_publication(**kwargs):
            await database.workflow_runs.update_workflow_run("wr_test", browser_session_id="pbs_replacement")
            return await mark_running(**kwargs)

        monkeypatch.setattr(svc, "mark_workflow_run_as_running", refresh_after_competing_publication)

    async def create(**kwargs):
        if case.startswith("refresh_") and kwargs["expected_browser_session_id"] != "pbs_replacement":
            raise service_module.WorkflowAttemptDispatchSuperseded("wr_test", 1)
        if case in {"replaced", "stale_fallback"}:
            async with database.Session() as session:
                await session.execute(
                    update(WorkflowRunAttemptModel).values(started_at=datetime.now(UTC) + timedelta(seconds=1))
                )
                await session.commit()
            if case == "stale_fallback":
                raise RuntimeError("recoverable provisioning failure")
            raise BrowserSessionCreditAdmissionRefusal()
        if case == "refused":
            raise BrowserSessionCreditAdmissionRefusal()
        if case == "fallback":
            raise RuntimeError("recoverable provisioning failure")
        # The manager commits the association before returning the created session.
        async with database.Session() as session:
            session.add(
                PersistentBrowserSessionModel(
                    persistent_browser_session_id="pbs_forced",
                    organization_id="o_test",
                    status="running",
                    runnable_type="forced_workflow_run",
                )
            )
            await session.commit()
        update_run = database.workflow_runs.update_workflow_run
        await update_run(kwargs["workflow_run_id"], browser_session_id="pbs_forced")
        if case == "assignment":

            async def reject_duplicate_assignment(*args: Any, **kwargs: Any) -> WorkflowRun:
                if kwargs.get("browser_session_id") == "pbs_forced":
                    raise RuntimeError("duplicate assignment failed")
                return await update_run(*args, **kwargs)

            monkeypatch.setattr(
                database.workflow_runs, "update_workflow_run", AsyncMock(side_effect=reject_duplicate_assignment)
            )
        return SimpleNamespace(persistent_browser_session_id="pbs_forced")

    app.PERSISTENT_SESSIONS_MANAGER.create_session.side_effect = create
    # Address and admitted reuse are setup controls; their separate attachment tests cover execution.
    if case in {"address", "reuse"}:
        assert "force_pending" not in (run.reuse_bound_key or "")
        app.PERSISTENT_SESSIONS_MANAGER.create_session.assert_not_awaited()
        return
    if case in {"replaced", "stale_fallback", "refresh_none", "refresh_supplied"}:
        with pytest.raises(service_module.WorkflowAttemptDispatchSuperseded):
            await _run_execute_workflow(svc, dispatch_claim_started_at=claim)
    elif case == "refused":
        await _run_execute_workflow(svc, dispatch_claim_started_at=claim)
    else:
        monkeypatch.setattr(svc, "_ensure_browser_session_lease", AsyncMock())
        with pytest.raises(asyncio.CancelledError):
            await _run_execute_workflow(svc, dispatch_claim_started_at=claim)
    if svc._background_tasks:
        await asyncio.gather(*list(svc._background_tasks))
    persisted = await database.workflow_runs.get_workflow_run("wr_test", "o_test")
    assert persisted is not None
    assert not any(c.args[0] == "FORCE_BROWSER_SESSION" for c in flag.await_args_list)
    if case in {"refused", "replaced", "stale_fallback", "refresh_none", "refresh_supplied"}:
        svc.auto_create_browser_session_if_needed.assert_not_awaited()
        svc._execute_workflow_blocks.assert_not_awaited()
        assert persisted.status == (WorkflowRunStatus.failed if case == "refused" else WorkflowRunStatus.running)
        if case.startswith("refresh_"):
            assert persisted.browser_session_id == "pbs_replacement"
            assert (
                app.PERSISTENT_SESSIONS_MANAGER.create_session.await_args.kwargs["expected_browser_session_id"]
                == original
            )
    else:
        assert persisted.reuse_bound_key == service_module.REUSE_ADMISSION_OFF_DISABLED
        assert svc.auto_create_browser_session_if_needed.await_count == 1
        session_id = persisted.browser_session_id
        assert session_id == ("pbs_forced" if case in {"on", "assignment"} else None)
        assert svc.auto_create_browser_session_if_needed.await_args.kwargs["browser_session_id"] == session_id
        assert app.WORKFLOW_CONTEXT_MANAGER.get_workflow_run_context("wr_test").browser_session_id == session_id
        attempts = await database.workflow_run_attempts.get_attempts("wr_test")
        assert attempts[0].pinned_browser_session_id == session_id
        svc._execute_workflow_blocks.assert_awaited_once()
        if case == "assignment":
            assert not any(
                "browser_session_id" in call.kwargs
                for call in database.workflow_runs.update_workflow_run.await_args_list
            )
        calls = app.PERSISTENT_SESSIONS_MANAGER.create_session.await_count
        for attempt in (1, 2):
            if attempt == 2:
                await database.workflow_runs.update_workflow_run("wr_test", status=WorkflowRunStatus.failed)
                await database.workflow_run_attempts.finalize_attempt(
                    "wr_test",
                    1,
                    status="failed",
                    failure_reason="retry preparation",
                    failure_category=None,
                    error_codes=[],
                    finished_at=datetime.now(UTC),
                    retry_decision=RETRY_DECISION_RETRY,
                    decision_reason="matched_rule",
                    next_attempt_at=None,
                )
                preparation = await service_module.prepare_next_attempt_result("wr_test", "o_test", 1)
                assert preparation.status == "inserted"
                assert preparation.pinned_browser_session_id == persisted.browser_session_id
                claim = await database.workflow_run_attempts.claim_prepared_attempt_execution("wr_test", "o_test", 2)
                assert claim is not None
            with pytest.raises(asyncio.CancelledError):
                await _run_execute_workflow(svc, attempt_number=attempt, dispatch_claim_started_at=claim)
        assert app.PERSISTENT_SESSIONS_MANAGER.create_session.await_count == calls == int(selected)


@pytest.mark.asyncio
@pytest.mark.parametrize("case", ["pin_cancelled", "cancelled", "timeout", "failed", "denied", "success"])
async def test_forced_retirement_claim_propagates_cancellation(monkeypatch, forced_session_setup, case):
    database, svc, workflow = forced_session_setup
    monkeypatch.setattr(app.AGENT_FUNCTION, "should_defer_workflow_browser_creation", AsyncMock(return_value=True))
    request = WorkflowRequestBody(data={})
    run = await svc.create_workflow_run(
        workflow_request=request,
        workflow_permanent_id=workflow.workflow_permanent_id,
        workflow_id=workflow.workflow_id,
        organization_id="o_test",
        workflow=workflow,
    )
    if case == "pin_cancelled":
        prepared_session = await app.PERSISTENT_SESSIONS_MANAGER.create_session(
            organization_id=run.organization_id, workflow_run_id=run.workflow_run_id
        )
        monkeypatch.setattr(app.PERSISTENT_SESSIONS_MANAGER, "create_session", AsyncMock(return_value=prepared_session))
    pin_error = asyncio.CancelledError() if case == "pin_cancelled" else RuntimeError("pin failed")
    entered, release = asyncio.Event(), asyncio.Event()

    async def pin(**kwargs):
        if case == "pin_cancelled":
            entered.set()
        raise pin_error

    monkeypatch.setattr(database.workflow_runs, "pin_browser_session_for_dispatch", AsyncMock(side_effect=pin))
    order, budgets = [], []
    timeout_context = None
    original_claim = database.workflow_runs.claim_browser_session_retirement

    def bounded_timeout(seconds):
        nonlocal timeout_context
        budgets.append(seconds)
        timeout_context = asyncio.timeout(None)
        return timeout_context

    async def claim(**kwargs):
        entered.set()
        order.append("entered")
        try:
            if case in {"cancelled", "timeout"}:
                if case == "timeout" and timeout_context is not None:
                    timeout_context.reschedule(asyncio.get_running_loop().time())
                await release.wait()
            if case == "failed":
                raise RuntimeError("claim failed")
            if case == "success":
                assert await original_claim(**kwargs)
                order.append("claimed")
                return True
            return False
        finally:
            order.append("drained")

    retirement = AsyncMock(side_effect=claim)
    monkeypatch.setattr(database.workflow_runs, "claim_browser_session_retirement", retirement)
    close = AsyncMock(side_effect=lambda *args, **kwargs: order.append("closed"))
    monkeypatch.setattr(app.PERSISTENT_SESSIONS_MANAGER, "close_session", close)
    monkeypatch.setattr(service_module, "asyncio", ScopedAsyncio(timeout=bounded_timeout))
    task = asyncio.create_task(
        svc._prepare_forced_browser_session(
            workflow=workflow,
            workflow_run=run,
            workflow_request=request,
            has_attempt_row=True,
            use_resolved_run_seed=True,
            on_finalization_denied=lambda: order.append("denied"),
        )
    )
    expected = (
        asyncio.CancelledError
        if case in {"pin_cancelled", "cancelled"}
        else TimeoutError
        if case == "timeout"
        else RuntimeError
    )
    try:
        await asyncio.wait_for(entered.wait(), timeout=5)
        if case == "cancelled":
            task.cancel()
        # A propagation bug leaves the task parked forever; the bound only has to outlast a GC pause (~1s).
        done, _ = await asyncio.wait({task}, timeout=10)
        assert task in done, "retirement claim did not propagate cancellation or timeout"
        with pytest.raises(expected) as raised:
            await task
        if case == "success":
            assert order == ["entered", "claimed", "drained", "closed"]
            assert service_module.BROWSER_RETIREMENT_DENIED_NOTE not in getattr(raised.value, "__notes__", ())
        else:
            assert order == (["denied"] if case == "pin_cancelled" else ["entered", "drained", "denied"])
            assert service_module.BROWSER_RETIREMENT_DENIED_NOTE in raised.value.__notes__
            close.assert_not_awaited()
        assert budgets == ([] if case == "pin_cancelled" else [5])
        assert retirement.await_count == int(case != "pin_cancelled")
    finally:
        release.set()
        if not task.done():
            task.cancel()
        await asyncio.gather(task, return_exceptions=True)


@pytest.mark.asyncio
@pytest.mark.parametrize("case", ["claim", "pin_cancelled", "orphan", "orphan_cancelled"])
async def test_superseded_forced_startup_cannot_pin_or_close_replacement(monkeypatch, forced_session_setup, case):
    database, svc, workflow = forced_session_setup
    pin_cancelled = case == "pin_cancelled"
    orphan = case.startswith("orphan")
    monkeypatch.setattr(app.PERSISTENT_SESSIONS_MANAGER, "close_session", AsyncMock())
    async with database.Session() as session:
        session.add(
            PersistentBrowserSessionModel(
                persistent_browser_session_id="pbs_existing", organization_id="o_test", status="running"
            )
        )
        await session.commit()
    run = await svc.create_workflow_run(
        workflow_request=WorkflowRequestBody(data={}, browser_session_id="pbs_existing"),
        workflow_permanent_id=workflow.workflow_permanent_id,
        workflow_id=workflow.workflow_id,
        organization_id="o_test",
        workflow=workflow,
    )
    claim = datetime(2026, 1, 1)
    async with database.Session() as session:
        row = await session.get(service_module.WorkflowRunModel, run.workflow_run_id)
        row.status, row.browser_session_id = "running", None
        attempt = await session.scalar(select(WorkflowRunAttemptModel))
        attempt.status, attempt.started_at, attempt.pinned_browser_session_id = "running", claim, None
        await session.commit()
    run = await database.workflow_runs.get_workflow_run(run.workflow_run_id, "o_test")

    async def finish_obsolete_start(**kwargs):
        assert kwargs["attempt_number"] == 1 and kwargs["dispatch_claim_started_at"] == claim
        async with database.Session() as session:
            row = await session.get(service_module.WorkflowRunModel, run.workflow_run_id)
            row.browser_session_id = "pbs_obsolete" if pin_cancelled else "pbs_replacement"
            attempt = await session.scalar(select(WorkflowRunAttemptModel))
            if pin_cancelled or orphan:
                session.add(
                    PersistentBrowserSessionModel(
                        persistent_browser_session_id="pbs_obsolete",
                        organization_id="o_test",
                        status="running",
                        runnable_id=run.workflow_run_id if pin_cancelled else None,
                        runnable_type="workflow_run" if pin_cancelled else "forced_workflow_run",
                        runnable_generation_id="new" if pin_cancelled else None,
                    )
                )
            else:
                attempt.started_at = claim + timedelta(seconds=1)
            if orphan:
                session.add(
                    PersistentBrowserSessionModel(
                        persistent_browser_session_id="pbs_replacement", organization_id="o_test", status="running"
                    )
                )
                attempt.pinned_browser_session_id = "pbs_replacement"
            await session.commit()
        return SimpleNamespace(persistent_browser_session_id="pbs_obsolete")

    app.PERSISTENT_SESSIONS_MANAGER.create_session.side_effect = finish_obsolete_start
    denied = []
    if pin_cancelled:
        monkeypatch.setattr(
            database.workflow_runs, "pin_browser_session_for_dispatch", AsyncMock(side_effect=asyncio.CancelledError())
        )
    if case == "orphan_cancelled":
        app.PERSISTENT_SESSIONS_MANAGER.close_session.side_effect = asyncio.CancelledError()
    with pytest.raises(
        asyncio.CancelledError if "cancelled" in case else service_module.WorkflowAttemptDispatchSuperseded
    ) as raised:
        await svc._prepare_forced_browser_session(
            workflow=workflow,
            workflow_run=run,
            workflow_request=WorkflowRequestBody(data={}),
            has_attempt_row=True,
            use_resolved_run_seed=True,
            attempt_number=1,
            dispatch_claim_started_at=claim,
            on_finalization_denied=lambda: denied.append(True),
        )
    assert denied and service_module.BROWSER_RETIREMENT_DENIED_NOTE in getattr(raised.value, "__notes__", ())
    persisted = await database.workflow_runs.get_workflow_run(run.workflow_run_id, "o_test")
    assert persisted.status == WorkflowRunStatus.running
    assert persisted.browser_session_id == ("pbs_obsolete" if pin_cancelled else "pbs_replacement")
    attempt = (await database.workflow_run_attempts.get_attempts(run.workflow_run_id))[0]
    assert attempt.pinned_browser_session_id == ("pbs_replacement" if orphan else None)
    if orphan:
        assert attempt.started_at == claim
        retired = await database.browser_sessions.get_persistent_browser_session("pbs_obsolete", "o_test")
        replacement = await database.browser_sessions.get_persistent_browser_session("pbs_replacement", "o_test")
        assert retired.close_requested_at is not None
        assert replacement.status == "running" and replacement.close_requested_at is None
        app.PERSISTENT_SESSIONS_MANAGER.close_session.assert_awaited_once_with(
            "o_test", "pbs_obsolete", reason=BrowserSessionCloseReason.aborted
        )
    else:
        app.PERSISTENT_SESSIONS_MANAGER.close_session.assert_not_awaited()


@pytest.mark.asyncio
async def test_guarded_binding_clear_uses_and_commits_held_dispatch_session(monkeypatch, forced_session_setup):
    database, svc, workflow = forced_session_setup
    claim = datetime(2026, 9, 23)
    async with database.Session() as session:
        session.add_all(
            [
                WorkflowRunModel(
                    workflow_run_id="wr_current",
                    workflow_id="wf_1",
                    workflow_permanent_id="wpid_test",
                    organization_id="o_test",
                    status="running",
                    started_at=claim,
                ),
                WorkflowRunAttemptModel(
                    workflow_run_id="wr_current",
                    organization_id="o_test",
                    attempt_number=1,
                    status="running",
                    started_at=claim,
                ),
                PersistentBrowserSessionModel(
                    persistent_browser_session_id="pbs_previous",
                    organization_id="o_test",
                    status="running",
                    runnable_type="workflow_run",
                    runnable_id="wr_previous",
                    runnable_generation_id="gen_previous",
                    bound_workflow_permanent_id="wpid_test",
                    bound_key="key_previous",
                    download_run_id="wr_previous",
                ),
            ]
        )
        await session.commit()
    browser = await database.browser_sessions.get_persistent_browser_session("pbs_previous", "o_test")
    assert browser is not None
    dispatch_session = svc._workflow_run_dispatch_session
    clear_binding = database.browser_sessions.clear_persistent_browser_session_binding
    held = None

    @asynccontextmanager
    async def track_dispatch_session(**kwargs):
        nonlocal held
        async with dispatch_session(**kwargs) as session:
            held = session
            yield session

    def no_nested_session():
        raise AssertionError("binding clear opened a second database session")

    async def clear_in_held_session(**kwargs):
        assert held is not None and held.in_transaction()
        assert kwargs.get("db_session") is held
        with monkeypatch.context() as scope:
            scope.setattr(database.browser_sessions, "Session", no_nested_session)
            return await clear_binding(**kwargs)

    monkeypatch.setattr(svc, "_workflow_run_dispatch_session", track_dispatch_session)
    monkeypatch.setattr(database.browser_sessions, "clear_persistent_browser_session_binding", clear_in_held_session)
    result = await svc._retire_reused_session_for_respawn(
        organization_id="o_test",
        workflow_run_id="wr_current",
        workflow_permanent_id=workflow.workflow_permanent_id,
        bound_key="key_previous",
        browser_session=browser,
        attempt_number=1,
        dispatch_claim_started_at=claim,
    )
    assert result is None
    persisted = await database.browser_sessions.get_persistent_browser_session("pbs_previous", "o_test")
    assert persisted is not None
    assert (persisted.bound_workflow_permanent_id, persisted.bound_key, persisted.download_run_id) == (None, None, None)
    assert (persisted.runnable_id, persisted.runnable_type, persisted.runnable_generation_id) == (
        "wr_previous",
        "workflow_run",
        "gen_previous",
    )