Skip to content

Automate browser based workflows with AI

116 KB
Raw Blame
"""Tests for the WORKFLOW_TASK_V3_AB run-level engine A/B: run eligibility, arm resolution
(idempotency, kill switch, fail-closed), and the invariant that the persisted engine on
workflow_run_blocks and the dispatched engine come from the same resolution.
"""

from __future__ import annotations

import asyncio
import contextlib
import json
import time
from collections.abc import Callable, Iterator
from datetime import UTC, datetime, timedelta, timezone
from pathlib import Path
from types import SimpleNamespace
from typing import Any
from unittest.mock import AsyncMock, MagicMock, patch

import pytest
from pydantic import BaseModel, ConfigDict, ValidationError

import skyvern.forge.sdk.workflow.models.block as block_module
import skyvern.webeye.cdp_download_interceptor as cdp_module
from skyvern.config import settings
from skyvern.exceptions import FailedToNavigateToUrl
from skyvern.forge import app
from skyvern.forge.sdk.core import skyvern_context
from skyvern.forge.sdk.core.skyvern_context import SkyvernContext
from skyvern.forge.sdk.db.agent_db import AgentDB
from skyvern.forge.sdk.db.enums import WorkflowRunTriggerType
from skyvern.forge.sdk.db.models import Base
from skyvern.forge.sdk.experimentation.billing_tier import BILLING_TIER_PROPERTY, BillingTier
from skyvern.forge.sdk.experimentation.providers import BaseExperimentationProvider, NoOpExperimentationProvider
from skyvern.forge.sdk.experimentation.workflow_block_engine import (
    DISABLE_TASK_V3_FLAG,
    WORKFLOW_TASK_V3_AB_FLAG,
    WORKFLOW_TASK_V3_AB_NEW_WORKFLOWS_FLAG,
    WOULD_BE_CODE_PROPERTY,
    WorkflowBlockEngineRouteReason,
    _as_utc,
    code_mode_displaced,
    effective_default_engine,
    resolve_workflow_block_engine_arm,
    workflow_block_engine_override,
)
from skyvern.forge.sdk.schemas.tasks import TaskStatus
from skyvern.forge.sdk.workflow.models.block import (
    ActionBlock,
    BaseTaskBlock,
    Block,
    BranchCondition,
    BranchEvaluationContext,
    CodeBlock,
    ConditionalBlock,
    ExtractionBlock,
    FileDownloadBlock,
    ForLoopBlock,
    HumanInteractionBlock,
    JinjaBranchCriteria,
    NavigationBlock,
    PromptBranchCriteria,
    TaskBlock,
    UrlBlock,
    V3AbIneligibleReason,
    ValidationBlock,
    WhileLoopBlock,
    _evaluate_prompt_branch_conditions_batch,
    get_all_blocks,
    pinned_v1_block_count,
    run_is_eligible_for_v3_ab,
    takes_default_engine,
    v3_ab_ineligibility_reason,
)
from skyvern.forge.sdk.workflow.models.workflow import (
    COPILOT_TEST_WORKFLOW_CREATOR,
    Workflow,
    WorkflowDefinition,
    WorkflowRunStatus,
)
from skyvern.forge.sdk.workflow.service import WorkflowService, workflow_definitions_differ
from skyvern.forge.sdk.workflow.workflow_definition_converter import convert_workflow_definition
from skyvern.forge.taskv3.goal_composition import render_block_context
from skyvern.forge.taskv3.tools import DownloadFailureWatch
from skyvern.schemas.run_enums import RunEngine
from skyvern.schemas.workflows import (
    BlockResult,
    BlockStatus,
    BlockType,
    NavigationBlockYAML,
    WorkflowCreateYAMLRequest,
    WorkflowDefinitionYAML,
    WorkflowStatus,
)
from skyvern.services import script_service
from skyvern.webeye.cdp_download_interceptor import CDPDownloadInterceptor, begin_replay_candidate_hold
from tests.unit._task_block_fakes import mock_block_execute_deps, output_parameter
from tests.unit._workflow_block_engine_fakes import (
    WORKFLOW_BLOCK_ENGINE_APP_TARGET,
    FakeExperimentationProvider,
    consulted_flags,
    resolve_arm,
)
from tests.unit.helpers import make_organization, make_task
from tests.unit.test_agent_task_v3 import (
    _make_block,
    _make_output_parameter,
    _run_execute_step_gate,
    stub_workflow_block_engine_app,
)
from tests.unit.test_block_description_caching import _block_result, _setup_mocks


@pytest.fixture
def scoped_context() -> Iterator[SkyvernContext]:
    context = SkyvernContext()
    skyvern_context.set(context)
    try:
        yield context
    finally:
        skyvern_context.reset()


@pytest.fixture(autouse=True)
def _no_ambient_v3_default_cutoff(monkeypatch: pytest.MonkeyPatch) -> None:
    # The rollout this file covers sets TASK_V3_DEFAULT_ENGINE_WORKFLOW_CUTOFF in the environment, and
    # Settings reads .env at import: without pinning it, every test here answers to whatever the
    # machine happens to export.
    monkeypatch.setattr(settings, "TASK_V3_DEFAULT_ENGINE_WORKFLOW_CUTOFF", None)
    monkeypatch.setattr(settings, "TASK_V3_CHOSEN_ENGINE_CUTOFF", None)


@pytest.fixture
def v3_default_cutoff(monkeypatch: pytest.MonkeyPatch, _no_ambient_v3_default_cutoff: None) -> datetime:
    # Ordered after the autouse pin above by requesting it, so the off-by-default pin cannot clobber
    # the cutoff a test asked for.
    cutoff = datetime(2026, 9, 15, tzinfo=UTC)
    monkeypatch.setattr(settings, "TASK_V3_DEFAULT_ENGINE_WORKFLOW_CUTOFF", cutoff)
    return cutoff


def test_mixed_eligibility_run_download_block_is_eligible(scoped_context: SkyvernContext) -> None:
    eligible_1 = _make_block(TaskBlock, label="t1")
    eligible_2 = _make_block(NavigationBlock, label="t2", navigation_goal="Apply to the job")
    download_block = _make_block(ActionBlock, label="dl", complete_on_download=True)
    blocks: list[BaseTaskBlock] = [eligible_1, eligible_2, download_block]

    assert run_is_eligible_for_v3_ab(blocks) is True


def test_mixed_eligibility_run_file_download_block_is_eligible(scoped_context: SkyvernContext) -> None:
    eligible_1 = _make_block(TaskBlock, label="t1")
    eligible_2 = _make_block(NavigationBlock, label="t2", navigation_goal="Apply to the job")
    file_download_block = _make_block(FileDownloadBlock, label="fd")
    blocks: list[BaseTaskBlock] = [eligible_1, eligible_2, file_download_block]

    assert run_is_eligible_for_v3_ab(blocks) is True


def test_mixed_eligibility_run_download_gated_validation_block_is_not_eligible(
    scoped_context: SkyvernContext,
) -> None:
    # A validation block never acts on the page, so it can't trigger the download it would
    # complete on (SKY-14905); it must disqualify the whole run from the v3 A/B.
    eligible_1 = _make_block(TaskBlock, label="t1")
    eligible_2 = _make_block(NavigationBlock, label="t2", navigation_goal="Apply to the job")
    download_gated_validation = _make_block(ValidationBlock, label="dlv", complete_on_download=True)
    blocks: list[BaseTaskBlock] = [eligible_1, eligible_2, download_gated_validation]

    assert run_is_eligible_for_v3_ab(blocks) is False


@pytest.mark.asyncio
async def test_mixed_eligibility_run_pins_whole_run_to_control(scoped_context: SkyvernContext) -> None:
    eligible_1 = _make_block(TaskBlock, label="t1")
    eligible_2 = _make_block(NavigationBlock, label="t2", navigation_goal="Apply to the job")
    ineligible_block = _make_block(TaskBlock, label="unsupported")
    ineligible_block.block_type = BlockType.WAIT
    blocks: list[BaseTaskBlock] = [eligible_1, eligible_2, ineligible_block]

    assert run_is_eligible_for_v3_ab(blocks) is False

    provider = FakeExperimentationProvider({WORKFLOW_TASK_V3_AB_FLAG: True})
    await resolve_arm(
        scoped_context,
        provider,
        workflow_run_id="wr_mixed",
        ineligibility_reason=V3AbIneligibleReason.unsupported_block,
    )

    for block in blocks:
        assert block.resolve_engine("wr_mixed") == RunEngine.skyvern_v1
    # An ineligible run never even asks the provider -- there is nothing to bucket.
    assert provider.calls == []


@pytest.mark.asyncio
async def test_explicit_block_engine_is_never_overridden_by_treatment_arm(scoped_context: SkyvernContext) -> None:
    provider = FakeExperimentationProvider({WORKFLOW_TASK_V3_AB_FLAG: True})
    await resolve_arm(scoped_context, provider, workflow_run_id="wr_pinned", ineligibility_reason=None)
    assert scoped_context.workflow_block_engine_override == RunEngine.skyvern_v3

    cua_block = _make_block(TaskBlock, label="cua", engine=RunEngine.openai_cua)
    v2_block = _make_block(TaskBlock, label="v2", engine=RunEngine.skyvern_v2)

    assert cua_block.resolve_engine("wr_pinned") == RunEngine.openai_cua
    assert v2_block.resolve_engine("wr_pinned") == RunEngine.skyvern_v2


@pytest.mark.parametrize("pinned_engine", [RunEngine.openai_cua, RunEngine.skyvern_v3])
def test_pinned_non_default_engine_block_disqualifies_the_run(pinned_engine: RunEngine) -> None:
    eligible = _make_block(TaskBlock, label="ok")
    # Pinned as-authored in both arms, but that leaves control mixed-engine, so it
    # disqualifies the whole run rather than being skipped. A v3 pin is the user opting
    # in explicitly, not a treatment exposure.
    pinned = _make_block(NavigationBlock, label="pinned", navigation_goal="Apply to the job", engine=pinned_engine)

    assert run_is_eligible_for_v3_ab([eligible, pinned]) is False


@pytest.mark.asyncio
async def test_all_eligible_run_resolves_every_block_to_treatment(scoped_context: SkyvernContext) -> None:
    blocks: list[BaseTaskBlock] = [
        _make_block(TaskBlock, label="t1"),
        _make_block(NavigationBlock, label="t2", navigation_goal="Apply to the job"),
        _make_block(ActionBlock, label="t3"),
    ]
    assert run_is_eligible_for_v3_ab(blocks) is True

    provider = FakeExperimentationProvider({WORKFLOW_TASK_V3_AB_FLAG: True})
    await resolve_arm(scoped_context, provider, workflow_run_id="wr_treatment", ineligibility_reason=None)

    for block in blocks:
        assert block.resolve_engine("wr_treatment") == RunEngine.skyvern_v3


@pytest.mark.asyncio
async def test_arm_resolved_once_per_run_survives_mid_run_flag_flip(scoped_context: SkyvernContext) -> None:
    provider = FakeExperimentationProvider({WORKFLOW_TASK_V3_AB_FLAG: True})
    await resolve_arm(scoped_context, provider, workflow_run_id="wr_once", ineligibility_reason=None)
    assert scoped_context.workflow_block_engine_override == RunEngine.skyvern_v3

    # Invalidate the provider's own 300s cache and flip the flag, so a second query would
    # return False if it actually reached the provider. The idempotency guard must still
    # short-circuit on context.workflow_block_engine_resolved_run_id before that happens.
    provider.invalidate_resolution_caches()
    provider.flags[WORKFLOW_TASK_V3_AB_FLAG] = False
    await resolve_arm(scoped_context, provider, workflow_run_id="wr_once", ineligibility_reason=None)

    assert scoped_context.workflow_block_engine_override == RunEngine.skyvern_v3


@pytest.mark.asyncio
async def test_different_run_id_on_same_context_reresolves_instead_of_inheriting(
    scoped_context: SkyvernContext,
) -> None:
    provider = FakeExperimentationProvider({WORKFLOW_TASK_V3_AB_FLAG: True})
    await resolve_arm(scoped_context, provider, workflow_run_id="wr_A", ineligibility_reason=None)
    assert workflow_block_engine_override("wr_A") == RunEngine.skyvern_v3

    provider.flags[WORKFLOW_TASK_V3_AB_FLAG] = False
    await resolve_arm(scoped_context, provider, workflow_run_id="wr_B", ineligibility_reason=None)

    assert workflow_block_engine_override("wr_B") is None
    # The pin moved to B: A must not read as still-treatment via a stale resolution.
    assert workflow_block_engine_override("wr_A") is None


@pytest.mark.asyncio
async def test_unresolved_run_id_reads_control_without_resolving(scoped_context: SkyvernContext) -> None:
    provider = FakeExperimentationProvider({WORKFLOW_TASK_V3_AB_FLAG: True})
    await resolve_arm(scoped_context, provider, workflow_run_id="wr_A", ineligibility_reason=None)
    assert workflow_block_engine_override("wr_A") == RunEngine.skyvern_v3

    # wr_B was never resolved (task_v2 / cached-script helper paths never call the resolver for
    # their run), so the reader must not fall back to A's pin or to a bare "is anything pinned".
    assert workflow_block_engine_override("wr_B") is None
    assert all(distinct_id == "wr_A" for _, distinct_id, _ in provider.calls)


@pytest.mark.asyncio
async def test_resolver_matches_execute_step_flag_contract(scoped_context: SkyvernContext) -> None:
    """Derives the expected DISABLE_TASK_V3 call from the real execute_step gate instead of a
    hardcoded literal, so a drift in agent.py's distinct_id or properties reds this test.
    """
    gate_provider = FakeExperimentationProvider()
    await _run_execute_step_gate(
        engine=RunEngine.skyvern_v3,
        task_block=_make_block(TaskBlock, label="contract"),
        experimentation_provider=gate_provider,
        workflow_run_id="wr_contract",
    )
    gate_disable_calls = [call for call in gate_provider.calls if call[0] == DISABLE_TASK_V3_FLAG]
    assert len(gate_disable_calls) == 1
    gate_call = gate_disable_calls[0]
    assert gate_call[1] == "wr_contract"
    # Pinned separately from the equality below: both callers now build this dict in one shared
    # place, so dropping it would keep them agreeing with each other while silently losing the
    # organization targeting the flag's release conditions are written against.
    assert gate_call[2] == {"organization_id": make_organization(datetime.now(UTC)).organization_id}

    resolver_provider = FakeExperimentationProvider({WORKFLOW_TASK_V3_AB_FLAG: True})
    await resolve_arm(
        scoped_context,
        resolver_provider,
        workflow_run_id="wr_contract",
        ineligibility_reason=None,
        organization_id=(gate_call[2] or {}).get("organization_id"),
        workflow_permanent_id="wpid_contract",
    )
    resolver_disable_calls = [call for call in resolver_provider.calls if call[0] == DISABLE_TASK_V3_FLAG]
    assert len(resolver_disable_calls) == 1
    assert resolver_disable_calls[0] == gate_call

    assert any(
        call[0] == WORKFLOW_TASK_V3_AB_FLAG and (call[2] or {}).get("workflow_permanent_id") == "wpid_contract"
        for call in resolver_provider.calls
    )


@pytest.mark.asyncio
async def test_disable_flag_wins_over_ab_flag(scoped_context: SkyvernContext) -> None:
    provider = FakeExperimentationProvider({WORKFLOW_TASK_V3_AB_FLAG: True, DISABLE_TASK_V3_FLAG: True})
    await resolve_arm(scoped_context, provider, workflow_run_id="wr_disabled", ineligibility_reason=None)

    assert scoped_context.workflow_block_engine_override is None
    block = _make_block(TaskBlock, label="disabled_block")
    assert block.resolve_engine("wr_disabled") == RunEngine.skyvern_v1


@pytest.mark.asyncio
async def test_provider_exception_fails_closed_to_control(scoped_context: SkyvernContext) -> None:
    provider = FakeExperimentationProvider(raise_error=True)
    resolution = await resolve_arm(scoped_context, provider, workflow_run_id="wr_err", ineligibility_reason=None)

    assert scoped_context.workflow_block_engine_override is None
    block = _make_block(TaskBlock, label="err_block")
    assert block.resolve_engine("wr_err") == RunEngine.skyvern_v1
    # An unrandomized control run has to be separable from a bucketed one in the same read.
    assert resolution.log["route_reason"] == WorkflowBlockEngineRouteReason.flag_error


@pytest.mark.asyncio
async def test_an_unevaluable_kill_switch_fails_closed_to_control(scoped_context: SkyvernContext) -> None:
    # Both PostHog providers swallow an evaluation error into None, which the boolean resolver reads
    # as "not disabled": the kill switch would fail open onto the A/B exactly when the provider is down.
    provider = FakeExperimentationProvider({WORKFLOW_TASK_V3_AB_FLAG: True}, strict_error_flags={DISABLE_TASK_V3_FLAG})
    resolution = await resolve_arm(scoped_context, provider, workflow_run_id="wr_kill_down", ineligibility_reason=None)

    assert _make_block(TaskBlock, label="kill_down").resolve_engine("wr_kill_down") == RunEngine.skyvern_v1
    assert resolution.log["route_reason"] == WorkflowBlockEngineRouteReason.flag_error


@pytest.mark.asyncio
async def test_a_failing_ab_evaluation_is_labelled_flag_error_not_bucketed_control(
    scoped_context: SkyvernContext,
) -> None:
    # A failed A/B evaluation and a real control bucket produce the same arm, so only route_reason
    # separates them. Reading a provider outage as flag_bucket_control mixes runs that were never
    # randomized into the control cell of every per-arm comparison, which is the one thing this field
    # exists to prevent. Only reachable through the strict resolver: the boolean one swallows the
    # error into None and bool()s it to a plain False.
    provider = FakeExperimentationProvider(strict_error_flags={WORKFLOW_TASK_V3_AB_FLAG})
    resolution = await resolve_arm(scoped_context, provider, workflow_run_id="wr_ab_down", ineligibility_reason=None)

    assert scoped_context.workflow_block_engine_override is None
    assert _make_block(TaskBlock, label="ab_down").resolve_engine("wr_ab_down") == RunEngine.skyvern_v1
    assert resolution.log["route_reason"] == WorkflowBlockEngineRouteReason.flag_error


@pytest.mark.asyncio
async def test_an_ab_flag_local_evaluation_cannot_answer_is_control_but_not_a_bucket(
    scoped_context: SkyvernContext,
) -> None:
    # The third A/B answer: None from local evaluation is neither a failure nor a bucket. A key the
    # snapshot has no row for, or a condition local evaluation cannot answer, leaves the run on
    # control without it ever having been randomized -- and the runbook builds this rule's control
    # cell out of flag_bucket_control, so labelling it that way pours never-randomized runs into
    # every per-arm comparison, the same defect flag_error exists to prevent. The arm is unchanged.
    provider = FakeExperimentationProvider(
        {WORKFLOW_TASK_V3_AB_FLAG: True}, unresolvable_flags={WORKFLOW_TASK_V3_AB_FLAG}
    )
    resolution = await resolve_arm(
        scoped_context, provider, workflow_run_id="wr_ab_undefined", ineligibility_reason=None
    )

    assert scoped_context.workflow_block_engine_override is None
    assert resolution.log["route_reason"] == WorkflowBlockEngineRouteReason.flag_undefined


@pytest.mark.asyncio
async def test_noop_provider_never_queried_and_leaves_engine_unchanged(scoped_context: SkyvernContext) -> None:
    provider = NoOpExperimentationProvider()
    spy = AsyncMock(wraps=provider._is_feature_enabled)
    with patch(WORKFLOW_BLOCK_ENGINE_APP_TARGET) as mock_app, patch.object(provider, "_is_feature_enabled", spy):
        mock_app.EXPERIMENTATION_PROVIDER = provider
        stub_workflow_block_engine_app(mock_app)
        await resolve_workflow_block_engine_arm(
            scoped_context,
            workflow_run_id="wr_noop",
            organization_id="org_1",
            workflow_permanent_id="wpid_1",
            workflow_status=WorkflowStatus.published,
            trigger_type=WorkflowRunTriggerType.api,
            ineligibility_reason=None,
            takes_default_engine=True,
        )

    spy.assert_not_called()
    assert scoped_context.workflow_block_engine_override is None
    block = _make_block(TaskBlock, label="noop_block")
    assert block.resolve_engine("wr_noop") == RunEngine.skyvern_v1


def test_non_task_blocks_ignored_but_nested_loop_task_blocks_considered() -> None:
    code_block = CodeBlock(label="code", output_parameter=_make_output_parameter("code"), code="pass")
    eligible = _make_block(TaskBlock, label="ok")

    flat_without_loop = get_all_blocks([code_block, eligible])
    assert run_is_eligible_for_v3_ab(flat_without_loop) is True

    ineligible_nested = _make_block(TaskBlock, label="nested_pinned", engine=RunEngine.openai_cua)
    loop = ForLoopBlock(
        label="loop",
        output_parameter=_make_output_parameter("loop"),
        loop_blocks=[ineligible_nested],
    )

    flat_with_loop = get_all_blocks([code_block, eligible, loop])
    assert ineligible_nested in flat_with_loop
    assert run_is_eligible_for_v3_ab(flat_with_loop) is False


def test_inert_blocks_do_not_disqualify_an_otherwise_eligible_run() -> None:
    # Nearly every workflow starts with a Go-to-URL block; treating it (or a trailing
    # HumanInteractionBlock) as a disqualifier would kill nearly all experiment traffic.
    url_block = _make_block(UrlBlock, label="goto", url="https://example.com")
    human_block = _make_block(HumanInteractionBlock, label="human")
    eligible = _make_block(TaskBlock, label="ok")

    assert run_is_eligible_for_v3_ab([url_block, human_block, eligible]) is True


def test_run_with_only_inert_or_non_task_blocks_is_not_eligible() -> None:
    code_block = CodeBlock(label="code", output_parameter=_make_output_parameter("code"), code="pass")
    url_block = _make_block(UrlBlock, label="goto", url="https://example.com")
    human_block = _make_block(HumanInteractionBlock, label="human")

    assert run_is_eligible_for_v3_ab([code_block, url_block, human_block]) is False


@pytest.mark.asyncio
async def test_inert_blocks_resolve_to_v1_in_a_treated_run(scoped_context: SkyvernContext) -> None:
    # Eligibility skips GOTO_URL/HumanInteraction as engine-inert, so resolve_engine must skip
    # them too -- otherwise their workflow_run_blocks rows claim an engine that never ran.
    provider = FakeExperimentationProvider({WORKFLOW_TASK_V3_AB_FLAG: True})
    await resolve_arm(scoped_context, provider, workflow_run_id="wr_inert", ineligibility_reason=None)
    assert scoped_context.workflow_block_engine_override == RunEngine.skyvern_v3

    url_block = _make_block(UrlBlock, label="goto", url="https://example.com")
    human_block = _make_block(HumanInteractionBlock, label="human")

    assert url_block.resolve_engine("wr_inert") == RunEngine.skyvern_v1
    assert human_block.resolve_engine("wr_inert") == RunEngine.skyvern_v1


@pytest.mark.asyncio
async def test_exclude_from_engine_ab_block_is_never_rerouted(scoped_context: SkyvernContext) -> None:
    provider = FakeExperimentationProvider({WORKFLOW_TASK_V3_AB_FLAG: True})
    await resolve_arm(scoped_context, provider, workflow_run_id="wr_excluded", ineligibility_reason=None)
    assert scoped_context.workflow_block_engine_override == RunEngine.skyvern_v3

    excluded_block = _make_block(ActionBlock, label="excluded")
    excluded_block._exclude_from_engine_ab = True
    assert excluded_block.resolve_engine("wr_excluded") == RunEngine.skyvern_v1


@pytest.mark.asyncio
async def test_execute_safe_persists_the_resolved_engine(scoped_context: SkyvernContext) -> None:
    scoped_context.workflow_block_engine_resolved_run_id = "wr_persist"
    scoped_context.workflow_block_engine_override = RunEngine.skyvern_v3
    block = _make_block(TaskBlock, label="persist_block")

    with (
        patch("skyvern.forge.sdk.workflow.models.block.app") as mock_app,
        patch.object(BaseTaskBlock, "execute", new_callable=AsyncMock, return_value=_block_result()),
        patch.object(Block, "_generate_workflow_run_block_description", new_callable=AsyncMock),
    ):
        _setup_mocks(mock_app)

        await block.execute_safe(workflow_run_id="wr_persist")

    persisted_engine = mock_app.DATABASE.observer.create_workflow_run_block.await_args.kwargs["engine"]
    assert persisted_engine == RunEngine.skyvern_v3


@pytest.mark.asyncio
@pytest.mark.parametrize(
    ("override", "script_mode", "fails", "seen"),
    [
        (RunEngine.skyvern_v3, False, False, 30.0),
        (RunEngine.skyvern_v3, False, True, 30.0),
        (None, False, False, 7.0),
        (RunEngine.skyvern_v3, True, False, 7.0),
    ],
    ids=["v3_block", "v3_block_that_fails", "v1_block_unchanged", "script_mode_unchanged"],
)
async def test_a_v3_block_scopes_its_download_timeout_to_the_whole_block(
    scoped_context: SkyvernContext,
    override: RunEngine | None,
    script_mode: bool,
    fails: bool,
    seen: float,
) -> None:
    # The browser's download listener reads this value when a download starts, including one the block's own
    # navigation starts before the task loop; it must be the outer value again once the block ends.
    scoped_context.workflow_block_engine_resolved_run_id = "wr_scope"
    scoped_context.workflow_block_engine_override = override
    scoped_context.script_mode = script_mode
    scoped_context.download_timeout = 7.0
    block = _make_block(TaskBlock, label="scope_block", download_timeout=30.0)
    observed: list[float | None] = []

    async def run_block(*_args: Any, **_kwargs: Any) -> BlockResult:
        observed.append(scoped_context.download_timeout)
        if fails:
            raise RuntimeError("block failed")
        return _block_result()

    with (
        patch("skyvern.forge.sdk.workflow.models.block.app") as mock_app,
        patch.object(BaseTaskBlock, "execute", new_callable=AsyncMock, side_effect=run_block),
        patch.object(Block, "_generate_workflow_run_block_description", new_callable=AsyncMock),
    ):
        _setup_mocks(mock_app)
        await block.execute_safe(workflow_run_id="wr_scope")

    assert observed == [seen]
    assert scoped_context.download_timeout == 7.0


class _EngineCaptured(BaseException):
    """Raised from the create_workflow_run_block mock once the dispatched engine is captured.

    Subclasses BaseException, not Exception, so it escapes the `except Exception` handlers in
    Block.execute_safe, WorkflowService._execute_block_via_agent_if_allowed, and
    WorkflowService._execute_single_block instead of being swallowed into a failed BlockResult.
    """


@pytest.mark.asyncio
async def test_base_task_block_execute_dispatches_with_the_resolved_engine(scoped_context: SkyvernContext) -> None:
    """Drives the real BaseTaskBlock.execute (not a mock of it) up to the app.agent.execute_step
    call. A revert of engine=self.resolve_engine(...) back to engine=self.engine only shows up
    here -- the persist-side test mocks BaseTaskBlock.execute out entirely.
    """
    scoped_context.workflow_block_engine_resolved_run_id = "wr_task_block_test"
    scoped_context.workflow_block_engine_override = RunEngine.skyvern_v3
    block = _make_block(TaskBlock, label="dispatch_block")

    captured: dict[str, Any] = {}

    async def _capture_engine_and_abort(**kwargs: Any) -> Any:
        captured["engine"] = kwargs.get("engine")
        raise _EngineCaptured()

    with mock_block_execute_deps(working_page_url="https://example.com/dashboard") as deps:
        deps.agent.execute_step = AsyncMock(side_effect=_capture_engine_and_abort)

        with pytest.raises(_EngineCaptured):
            await block.execute(
                workflow_run_id="wr_task_block_test",
                workflow_run_block_id="wrb_test",
                organization_id="o_test",
            )

    assert captured["engine"] == RunEngine.skyvern_v3


@pytest.mark.asyncio
async def test_a_retry_after_a_step_engine_fallback_runs_v3_under_a_v3_row(scoped_context: SkyvernContext) -> None:
    # Two attempts share one row. Attempt 1 falls back to the step engine (an unevaluable kill switch),
    # which relabels the row v1 and fails; attempt 2 dispatches the pin on v3 again.
    block = _make_block(TaskBlock, label="pinned_retry", engine=RunEngine.skyvern_v3, max_retries=1)
    row = {"engine": RunEngine.skyvern_v3.value}
    dispatches: list[tuple[RunEngine, str]] = []

    async def _update_row(**kwargs: Any) -> Any:
        if kwargs.get("engine") is not None:
            row["engine"] = kwargs["engine"]
        return MagicMock()

    async def _execute_step(**kwargs: Any) -> Any:
        dispatches.append((kwargs["engine"], row["engine"]))
        if len(dispatches) == 1:
            row["engine"] = RunEngine.skyvern_v1.value
            return None
        raise _EngineCaptured()

    now = datetime.now(UTC)
    failed_task = make_task(now, make_organization(now), status=TaskStatus.failed, failure_reason="step failed")
    with mock_block_execute_deps(working_page_url="https://example.com/dashboard") as deps:
        deps.agent.execute_step = AsyncMock(side_effect=_execute_step)
        deps.observer_db.update_workflow_run_block = AsyncMock(side_effect=_update_row)
        deps.tasks_db.get_task = AsyncMock(return_value=failed_task)
        with (
            patch(
                "skyvern.forge.sdk.workflow.models.block.app.STORAGE",
                new=MagicMock(
                    get_current_attempt_downloaded_files=AsyncMock(return_value=[]),
                    get_downloaded_file_signature_aliases=MagicMock(return_value=()),
                ),
                create=True,
            ),
            patch(
                "skyvern.forge.sdk.workflow.models.block.app.WORKFLOW_SERVICE.get_recent_task_screenshot_artifacts",
                new=AsyncMock(return_value=[]),
            ),
            patch(
                "skyvern.forge.sdk.workflow.models.block.app.WORKFLOW_SERVICE.get_recent_workflow_screenshot_artifacts",
                new=AsyncMock(return_value=[]),
            ),
            pytest.raises(_EngineCaptured),
        ):
            await block.execute(
                workflow_run_id="wr_task_block_test",
                workflow_run_block_id="wrb_test",
                organization_id="o_test",
            )

    assert dispatches == [
        (RunEngine.skyvern_v3, RunEngine.skyvern_v3.value),
        (RunEngine.skyvern_v3, RunEngine.skyvern_v3.value),
    ]


async def _persisted_engine_from_execute_workflow_blocks(
    monkeypatch: pytest.MonkeyPatch,
    provider: BaseExperimentationProvider,
    *,
    workflow_status: WorkflowStatus = WorkflowStatus.published,
    billing_tier: BillingTier = BillingTier.UNKNOWN,
    first_version_created_at: datetime | None = None,
    trigger_type: WorkflowRunTriggerType | None = WorkflowRunTriggerType.api,
    block: BaseTaskBlock | None = None,
    script: Any = None,
    code_gen: bool | None = None,
) -> RunEngine:
    """Drives the real WorkflowService._execute_workflow_blocks -> _execute_single_block ->
    Block.execute_safe chain for a single eligible TaskBlock and returns the engine it persisted,
    forcing only the experimentation provider, the billing tier and the workflow's birth timestamp.
    Everything the arm decision depends on is therefore supplied by the real caller, not by the
    resolver's own test seam.
    """
    monkeypatch.setattr(app, "EXPERIMENTATION_PROVIDER", provider)
    monkeypatch.setattr(app.AGENT_FUNCTION, "resolve_billing_tier", AsyncMock(return_value=billing_tier))
    monkeypatch.setattr(
        app.DATABASE.workflows,
        "get_workflow_permanent_id_created_at",
        AsyncMock(return_value=first_version_created_at),
    )

    block = block or _make_block(TaskBlock, label="e2e_block")
    workflow = MagicMock()
    workflow.workflow_definition = WorkflowDefinition(parameters=[], blocks=[block])
    workflow.status = workflow_status

    workflow_run = MagicMock()
    workflow_run.workflow_run_id = "wr_e2e"
    workflow_run.workflow_permanent_id = "wpid_e2e"
    workflow_run.organization_id = "org_e2e"
    workflow_run.run_with = None
    workflow_run.retried_from_workflow_run_id = None
    workflow_run.trigger_type = trigger_type
    workflow_run.code_gen = code_gen

    organization = MagicMock()
    organization.organization_id = "org_e2e"

    # update_workflow_run_if_not_final's return value feeds datetime arithmetic in
    # mark_workflow_run_as_running; it needs real datetimes, not MagicMock, to avoid an
    # unrelated crash before execution ever reaches the resolver.
    now = datetime.now(UTC)
    running_run = MagicMock(organization_id="org_e2e", started_at=now, created_at=now, finished_at=None)
    monkeypatch.setattr(app.DATABASE.workflow_runs, "get_workflow_run", AsyncMock(return_value=None))
    monkeypatch.setattr(
        app.DATABASE.workflow_runs, "update_workflow_run_if_not_final", AsyncMock(return_value=running_run)
    )
    monkeypatch.setattr(app.WORKFLOW_CONTEXT_MANAGER, "get_workflow_run_context", MagicMock(return_value=MagicMock()))
    monkeypatch.setattr(app.WORKFLOW_CONTEXT_MANAGER, "register_block_parameters_for_workflow_run", AsyncMock())

    captured: dict[str, Any] = {}

    async def _capture_engine_and_abort(**kwargs: Any) -> Any:
        captured["engine"] = kwargs.get("engine")
        raise _EngineCaptured()

    monkeypatch.setattr(
        app.DATABASE.observer, "create_workflow_run_block", AsyncMock(side_effect=_capture_engine_and_abort)
    )

    service = WorkflowService()
    monkeypatch.setattr(service, "should_run_script", AsyncMock(return_value=script is not None))

    with pytest.raises(_EngineCaptured):
        await service._execute_workflow_blocks(
            workflow=workflow,
            workflow_run=workflow_run,
            organization=organization,
            script=script,
        )

    return captured["engine"]


@pytest.mark.asyncio
async def test_execute_workflow_blocks_pins_the_context_execute_safe_reads_from(
    scoped_context: SkyvernContext,
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    """If resolve_workflow_block_engine_arm pinned the arm on a different SkyvernContext object
    than the one execute_safe's resolve_engine() reads back from, this fails: the persisted engine
    would be skyvern_v1 instead of skyvern_v3.
    """
    provider = FakeExperimentationProvider({WORKFLOW_TASK_V3_AB_FLAG: True})

    engine = await _persisted_engine_from_execute_workflow_blocks(monkeypatch, provider)

    assert engine == RunEngine.skyvern_v3


@pytest.mark.parametrize(
    ("workflow_status", "expected_engine"),
    [
        (WorkflowStatus.auto_generated, RunEngine.skyvern_v1),
        (WorkflowStatus.published, RunEngine.skyvern_v3),
    ],
)
@pytest.mark.asyncio
async def test_execute_workflow_blocks_hands_the_resolver_the_running_versions_status(
    scoped_context: SkyvernContext,
    monkeypatch: pytest.MonkeyPatch,
    v3_default_cutoff: datetime,
    workflow_status: WorkflowStatus,
    expected_engine: RunEngine,
) -> None:
    # The per-call exclusion is only as real as the status the caller passes. Both cases are the same
    # new self-serve workflow with the percentage knob off, so the only thing deciding the engine is
    # workflow.status reaching the resolver: a caller that stopped passing it, or passed the
    # workflow_run instead, would take the login/download/credential-test/SDK endpoints' per-call
    # workflows onto v3 with it.
    provider = FakeExperimentationProvider({WORKFLOW_TASK_V3_AB_FLAG: False})

    engine = await _persisted_engine_from_execute_workflow_blocks(
        monkeypatch,
        provider,
        workflow_status=workflow_status,
        billing_tier=BillingTier.SELF_SERVE,
        first_version_created_at=v3_default_cutoff.replace(tzinfo=None) + timedelta(days=1),
    )

    assert engine == expected_engine


@pytest.mark.parametrize(
    ("trigger_type", "expected_engine"),
    [
        (WorkflowRunTriggerType.job_recipe_apply, RunEngine.skyvern_v1),
        (WorkflowRunTriggerType.job_recipe_extract, RunEngine.skyvern_v1),
        (WorkflowRunTriggerType.api, RunEngine.skyvern_v3),
    ],
)
@pytest.mark.asyncio
async def test_execute_workflow_blocks_hands_the_resolver_the_runs_trigger_type(
    scoped_context: SkyvernContext,
    monkeypatch: pytest.MonkeyPatch,
    v3_default_cutoff: datetime,
    trigger_type: WorkflowRunTriggerType,
    expected_engine: RunEngine,
) -> None:
    # The job-recipe endpoints run a brand-new published workflow_permanent_id per request, so the
    # status the caller passes cannot exclude them and only the run's trigger kind can. All three
    # cases are the same new self-serve published workflow with the percentage knob off, so the only
    # thing deciding the engine is workflow_run.trigger_type reaching the resolver: a caller that
    # stopped passing it would force-route every recipe call to v3, unrandomized.
    provider = FakeExperimentationProvider({WORKFLOW_TASK_V3_AB_FLAG: False})

    engine = await _persisted_engine_from_execute_workflow_blocks(
        monkeypatch,
        provider,
        workflow_status=WorkflowStatus.published,
        billing_tier=BillingTier.SELF_SERVE,
        first_version_created_at=v3_default_cutoff.replace(tzinfo=None) + timedelta(days=1),
        trigger_type=trigger_type,
    )

    assert engine == expected_engine


def _eligible_mix_blocks() -> list[BaseTaskBlock]:
    return [
        _make_block(TaskBlock, label="t1"),
        _make_block(NavigationBlock, label="t2", navigation_goal="Apply to the job"),
        _make_block(ActionBlock, label="t3"),
    ]


def _pinned_engine_blocks() -> list[BaseTaskBlock]:
    eligible = _make_block(TaskBlock, label="ok")
    pinned = _make_block(
        NavigationBlock, label="pinned", navigation_goal="Apply to the job", engine=RunEngine.skyvern_v2
    )
    return [eligible, pinned]


def _unsupported_block_type_blocks() -> list[BaseTaskBlock]:
    # No shipped BaseTaskBlock subclass declares an unsupported, non-inert block_type today, so
    # force one to exercise the branch _task_block_supports_v3 exists to guard.
    unsupported = _make_block(TaskBlock, label="unsupported")
    unsupported.block_type = BlockType.WAIT
    return [unsupported]


def _download_gated_validation_blocks() -> list[BaseTaskBlock]:
    return [_make_block(ValidationBlock, label="dlv", complete_on_download=True)]


def _totp_blocks() -> list[BaseTaskBlock]:
    # The multi-block shape the removed gate excluded: a verification-URL block alongside an ordinary
    # one. v3 carries the verification source per block, so neither block disqualifies the run.
    return [
        _make_block(TaskBlock, label="signin", totp_verification_url="https://example.com/otp"),
        _make_block(NavigationBlock, label="after_signin", navigation_goal="Apply to the job"),
    ]


def _no_reroutable_blocks() -> list[BaseTaskBlock]:
    code_block = CodeBlock(label="code", output_parameter=_make_output_parameter("code"), code="pass")
    url_block = _make_block(UrlBlock, label="goto", url="https://example.com")
    human_block = _make_block(HumanInteractionBlock, label="human")
    return [code_block, url_block, human_block]


def _conditional_block(criteria: PromptBranchCriteria | JinjaBranchCriteria) -> ConditionalBlock:
    return ConditionalBlock(
        label="cond",
        output_parameter=_make_output_parameter("cond"),
        branch_conditions=[
            BranchCondition(criteria=criteria, next_block_label="a"),
            BranchCondition(is_default=True, next_block_label="b"),
        ],
    )


def _prompt_branch_conditional_only_blocks() -> list[Block]:
    return [*_no_reroutable_blocks(), _conditional_block(PromptBranchCriteria(expression="the page shows an error"))]


def _jinja_only_conditional_blocks() -> list[Block]:
    # BranchCondition's validator re-types criteria from expression shape: only a PURE jinja
    # expression stays JinjaBranchCriteria; jinja mixed with bare text re-types to prompt.
    return [*_no_reroutable_blocks(), _conditional_block(JinjaBranchCriteria(expression="{{ x == 'y' }}"))]


def _while_loop(criteria: PromptBranchCriteria | JinjaBranchCriteria) -> WhileLoopBlock:
    body = CodeBlock(label="loop_body", output_parameter=_make_output_parameter("loop_body"), code="pass")
    return WhileLoopBlock(
        label="while",
        output_parameter=_make_output_parameter("while"),
        condition=criteria,
        loop_blocks=[body],
    )


def _prompt_condition_while_loop_only_blocks() -> list[Block]:
    return [*_no_reroutable_blocks(), _while_loop(PromptBranchCriteria(expression="the page shows an error"))]


def _jinja_condition_while_loop_blocks() -> list[Block]:
    return [*_no_reroutable_blocks(), _while_loop(JinjaBranchCriteria(expression="{{ x == 'y' }}"))]


@pytest.mark.parametrize(
    ("blocks_factory", "expected_reason"),
    [
        (_pinned_engine_blocks, V3AbIneligibleReason.pinned_engine),
        (_unsupported_block_type_blocks, V3AbIneligibleReason.unsupported_block),
        (_download_gated_validation_blocks, V3AbIneligibleReason.unsupported_block),
        (_totp_blocks, None),
        (_no_reroutable_blocks, V3AbIneligibleReason.no_reroutable_blocks),
        (_eligible_mix_blocks, None),
        (_prompt_branch_conditional_only_blocks, None),
        (_jinja_only_conditional_blocks, V3AbIneligibleReason.no_reroutable_blocks),
        (_prompt_condition_while_loop_only_blocks, None),
        (_jinja_condition_while_loop_blocks, V3AbIneligibleReason.no_reroutable_blocks),
    ],
    ids=[
        "pinned_engine",
        "unsupported_block_type",
        "unsupported_block_validation_download",
        "totp_verification_url_is_admitted",
        "no_reroutable_blocks",
        "eligible",
        "prompt_branch_conditional_is_a_reroutable_surface",
        "jinja_only_conditional_stays_invisible",
        "prompt_condition_while_loop_is_a_reroutable_surface",
        "jinja_condition_while_loop_stays_invisible",
    ],
)
def test_v3_ab_ineligibility_reason_maps_each_disqualifier(
    blocks_factory: Callable[[], list[BaseTaskBlock]],
    expected_reason: V3AbIneligibleReason | None,
) -> None:
    blocks = blocks_factory()
    reason = v3_ab_ineligibility_reason(blocks)
    assert reason == expected_reason
    # run_is_eligible_for_v3_ab must stay a thin wrapper over the same decision.
    assert run_is_eligible_for_v3_ab(blocks) is (expected_reason is None)


@pytest.mark.asyncio
async def test_resolver_logs_the_ineligibility_reason(scoped_context: SkyvernContext) -> None:
    provider = FakeExperimentationProvider({WORKFLOW_TASK_V3_AB_FLAG: True})
    with (
        patch(WORKFLOW_BLOCK_ENGINE_APP_TARGET) as mock_app,
        patch("skyvern.forge.sdk.experimentation.workflow_block_engine.LOG") as mock_log,
    ):
        mock_app.EXPERIMENTATION_PROVIDER = provider
        stub_workflow_block_engine_app(mock_app)
        await resolve_workflow_block_engine_arm(
            scoped_context,
            workflow_run_id="wr_logged",
            organization_id="org_1",
            workflow_permanent_id="wpid_1",
            workflow_status=WorkflowStatus.published,
            trigger_type=WorkflowRunTriggerType.api,
            ineligibility_reason=V3AbIneligibleReason.pinned_engine,
            takes_default_engine=True,
        )

    assert scoped_context.workflow_block_engine_override is None
    resolved_call = mock_log.info.call_args
    assert resolved_call.args[0] == "Resolved workflow-block engine arm"
    assert resolved_call.kwargs["arm"] == "control"
    assert resolved_call.kwargs["ineligibility_reason"] == V3AbIneligibleReason.pinned_engine
    # The provider is never even queried for an ineligible run -- nothing to bucket.
    assert provider.calls == []


def _branch_eval_stub(captured_engines: list[RunEngine], workflow_run_id: str) -> Any:
    async def _capture_engine(self: ExtractionBlock, *args: Any, **kwargs: Any) -> BlockResult:
        # The label prefix is the telemetry handle that lets reads slice branch-eval outcomes
        # per arm (the spawned task takes it as its title); pinned here so it can't silently drift.
        assert self.label.startswith("prompt_branch_eval_")
        captured_engines.append(self.resolve_engine(workflow_run_id))
        return BlockResult(
            success=True,
            output_parameter=self.output_parameter,
            output_parameter_value={"evaluations": [{"condition_index": 1, "reasoning": "ok", "result": True}]},
            failure_reason=None,
        )

    return _capture_engine


@pytest.mark.asyncio
async def test_branch_eval_synthetic_block_honors_v3_override(scoped_context: SkyvernContext) -> None:
    """The prompt-branch batch evaluator's synthetic ExtractionBlock honors the run-level engine
    override like an authored block: a treated run evaluates branch conditions on its arm."""
    scoped_context.workflow_block_engine_resolved_run_id = "wr_branch_eval_v3"
    scoped_context.workflow_block_engine_override = RunEngine.skyvern_v3

    captured_engines: list[RunEngine] = []
    branch = BranchCondition(
        criteria=PromptBranchCriteria(expression="user selected premium plan"), next_block_label="x"
    )
    evaluation_context = BranchEvaluationContext(workflow_run_context=None, template_renderer=lambda expr: expr)
    evaluation_context.build_llm_safe_context_snapshot = MagicMock(return_value={})  # type: ignore[method-assign]

    with patch.object(ExtractionBlock, "execute", _branch_eval_stub(captured_engines, "wr_branch_eval_v3")):
        await _evaluate_prompt_branch_conditions_batch(
            log_label="cond",
            branches=[branch],
            evaluation_context=evaluation_context,
            workflow_run_id="wr_branch_eval_v3",
            workflow_run_block_id="wrb",
            organization_id="org_1",
            browser_session_id=None,
            workflow_id="wf_1",
        )

    assert captured_engines == [RunEngine.skyvern_v3]


@pytest.mark.asyncio
async def test_branch_eval_synthetic_block_gets_no_extraction_report_framing(scoped_context: SkyvernContext) -> None:
    """The branch evaluator's block has no navigation goal, so without its marker the extraction-report
    framing ("absent fields are null, finish completed") would reach it. Its own prompt
    judges conditions from their text, and a null result parses as False: a silently wrong branch (SKY-16398)."""
    scoped_context.workflow_block_engine_resolved_run_id = "wr_branch_eval_framing"
    scoped_context.workflow_block_engine_override = RunEngine.skyvern_v3
    captured: list[ExtractionBlock] = []
    stub = _branch_eval_stub([], "wr_branch_eval_framing")

    async def _capture_block(self: ExtractionBlock, *args: Any, **kwargs: Any) -> BlockResult:
        captured.append(self)
        return await stub(self, *args, **kwargs)

    branch = BranchCondition(
        criteria=PromptBranchCriteria(expression="user selected premium plan"), next_block_label="x"
    )
    evaluation_context = BranchEvaluationContext(workflow_run_context=None, template_renderer=lambda expr: expr)
    evaluation_context.build_llm_safe_context_snapshot = MagicMock(return_value={})  # type: ignore[method-assign]

    with patch.object(ExtractionBlock, "execute", _capture_block):
        await _evaluate_prompt_branch_conditions_batch(
            log_label="cond",
            branches=[branch],
            evaluation_context=evaluation_context,
            workflow_run_id="wr_branch_eval_framing",
            workflow_run_block_id="wrb",
            organization_id="org_1",
            browser_session_id=None,
            workflow_id="wf_1",
        )

    (block,) = captured
    now = datetime.now(UTC)
    task = make_task(now, make_organization(now), navigation_goal=None, data_extraction_goal=block.data_extraction_goal)
    assert "This block only reads the page" not in render_block_context(task, block)


@pytest.mark.asyncio
async def test_branch_eval_synthetic_block_defaults_to_v1_without_override(scoped_context: SkyvernContext) -> None:
    """With no run-level override pinned at all, the synthetic ExtractionBlock resolves to
    skyvern_v1 -- control runs stay on v1."""
    scoped_context.workflow_block_engine_resolved_run_id = "wr_branch_eval_control"
    scoped_context.workflow_block_engine_override = None

    captured_engines: list[RunEngine] = []
    branch = BranchCondition(
        criteria=PromptBranchCriteria(expression="user selected premium plan"), next_block_label="x"
    )
    evaluation_context = BranchEvaluationContext(workflow_run_context=None, template_renderer=lambda expr: expr)
    evaluation_context.build_llm_safe_context_snapshot = MagicMock(return_value={})  # type: ignore[method-assign]

    with patch.object(ExtractionBlock, "execute", _branch_eval_stub(captured_engines, "wr_branch_eval_control")):
        await _evaluate_prompt_branch_conditions_batch(
            log_label="cond",
            branches=[branch],
            evaluation_context=evaluation_context,
            workflow_run_id="wr_branch_eval_control",
            workflow_run_block_id="wrb",
            organization_id="org_1",
            browser_session_id=None,
            workflow_id="wf_1",
        )

    assert captured_engines == [RunEngine.skyvern_v1]


@pytest.mark.asyncio
async def test_script_create_site_stamps_the_script_run_marker(scoped_context: SkyvernContext) -> None:
    """A cached-script block never resolves an engine, so `engine IS NULL` alone cannot separate
    a script-executed block from an agent-executed one whose engine was never written. The script
    create-site stamps `script_run` at creation, making (engine NULL, script_run NULL) a genuine
    anomaly instead of an ordinary script run.
    """
    scoped_context.workflow_run_id = "wr_script_marker"
    scoped_context.organization_id = "o_test"

    with patch("skyvern.services.script_service.app") as mock_app:
        mock_app.DATABASE.observer.create_workflow_run_block = AsyncMock(
            return_value=SimpleNamespace(workflow_run_block_id="wrb_script_marker")
        )
        mock_app.DATABASE.observer.update_workflow_run_block = AsyncMock()
        mock_app.DATABASE.tasks.create_task = AsyncMock(
            return_value=SimpleNamespace(task_id="tsk_script_marker", workflow_run_id="wr_script_marker")
        )
        mock_app.DATABASE.tasks.create_step = AsyncMock(return_value=SimpleNamespace(step_id="stp_script_marker"))
        mock_app.BROWSER_MANAGER.get_for_workflow_run.return_value = None
        mock_app.WORKFLOW_CONTEXT_MANAGER.get_workflow_run_context.return_value.cancel_failure_evidence_capture = (
            AsyncMock()
        )
        mock_app.AGENT_FUNCTION.is_recipe_step_attempt = AsyncMock(return_value=False)
        mock_app.AGENT_FUNCTION.validate_step_execution = AsyncMock()
        mock_app.DATABASE.tasks.update_step = AsyncMock(return_value=SimpleNamespace(step_id="stp_script_marker"))

        block_id, task_id, step_id = await script_service._create_workflow_block_run_and_task(
            block_type=BlockType.NAVIGATION,
            label="nav",
        )

    # Everything after the create call is wrapped in a try/except that collapses to
    # (None, None, None); asserting the ids proves the whole path ran rather than bailing early.
    assert (block_id, task_id, step_id) == ("wrb_script_marker", "tsk_script_marker", "stp_script_marker")
    create_kwargs = mock_app.DATABASE.observer.create_workflow_run_block.await_args.kwargs
    assert create_kwargs["ai_fallback_triggered"] is False
    assert create_kwargs.get("engine") is None


@pytest.mark.asyncio
async def test_agent_create_site_leaves_the_script_run_marker_unset(scoped_context: SkyvernContext) -> None:
    """The other half of the discriminator: an agent-executed block must NOT carry the marker,
    or `script_run IS NOT NULL` stops meaning "script-executed". A forward guard on the agent
    create-site, which this change deliberately leaves alone -- it passes with or without it."""
    scoped_context.workflow_block_engine_resolved_run_id = "wr_agent_marker"
    scoped_context.workflow_block_engine_override = RunEngine.skyvern_v3
    block = _make_block(TaskBlock, label="agent_marker_block")

    with (
        patch("skyvern.forge.sdk.workflow.models.block.app") as mock_app,
        patch.object(BaseTaskBlock, "execute", new_callable=AsyncMock, return_value=_block_result()),
        patch.object(Block, "_generate_workflow_run_block_description", new_callable=AsyncMock),
    ):
        _setup_mocks(mock_app)

        await block.execute_safe(workflow_run_id="wr_agent_marker")

    create_kwargs = mock_app.DATABASE.observer.create_workflow_run_block.await_args.kwargs
    assert create_kwargs["engine"] == RunEngine.skyvern_v3
    assert create_kwargs.get("ai_fallback_triggered") is None


@pytest.mark.asyncio
async def test_arm_offers_the_billing_tier_to_the_ab_flag_but_not_the_kill_switch(
    scoped_context: SkyvernContext,
) -> None:
    # The tier is what lets one release condition hold enterprise and self-serve at different
    # percentages. It must stay off DISABLE_TASK_V3: this resolver and the dispatch gate both
    # evaluate that flag through task_v3_disabled, and the provider caches on
    # (flag, distinct_id, properties), so adding a property on one side only would let the kill
    # switch answer differently for the same run.
    provider = FakeExperimentationProvider({WORKFLOW_TASK_V3_AB_FLAG: True})
    with patch(WORKFLOW_BLOCK_ENGINE_APP_TARGET) as mock_app:
        mock_app.EXPERIMENTATION_PROVIDER = provider
        stub_workflow_block_engine_app(mock_app, billing_tier=BillingTier.SELF_SERVE)
        await resolve_workflow_block_engine_arm(
            scoped_context,
            workflow_run_id="wr_tier",
            organization_id="org_1",
            workflow_permanent_id="wpid_1",
            workflow_status=WorkflowStatus.published,
            trigger_type=WorkflowRunTriggerType.api,
            ineligibility_reason=None,
            takes_default_engine=True,
        )

    by_flag = {flag: properties or {} for flag, _distinct_id, properties in provider.calls}
    assert by_flag[WORKFLOW_TASK_V3_AB_FLAG][BILLING_TIER_PROPERTY] == "self_serve"
    assert BILLING_TIER_PROPERTY not in by_flag[DISABLE_TASK_V3_FLAG]
    assert scoped_context.workflow_block_engine_override == RunEngine.skyvern_v3


@pytest.mark.asyncio
async def test_a_run_that_never_reaches_the_ab_pays_for_no_tier_lookup(scoped_context: SkyvernContext) -> None:
    # An ineligible run reads no flag, so a tier read would be pure cost on a path that cannot use
    # it, and a logged tier would claim the run was bucketed on one. The same holds for a run the
    # kill switch stops: a Redis or pooler incident is exactly when that switch gets flipped, and
    # this resolver holds a lock while it runs.
    provider = FakeExperimentationProvider({WORKFLOW_TASK_V3_AB_FLAG: True})
    for reason, run_id in ((V3AbIneligibleReason.unsupported_block, "wr_inelig"), (None, "wr_killed")):
        provider.flags[DISABLE_TASK_V3_FLAG] = reason is None
        tier = AsyncMock(return_value=BillingTier.ENTERPRISE)
        with (
            patch(WORKFLOW_BLOCK_ENGINE_APP_TARGET) as mock_app,
            patch("skyvern.forge.sdk.experimentation.workflow_block_engine.LOG") as mock_log,
        ):
            mock_app.EXPERIMENTATION_PROVIDER = provider
            mock_app.AGENT_FUNCTION.resolve_billing_tier = tier
            await resolve_workflow_block_engine_arm(
                scoped_context,
                workflow_run_id=run_id,
                organization_id="org_1",
                workflow_permanent_id="wpid_1",
                workflow_status=WorkflowStatus.published,
                trigger_type=WorkflowRunTriggerType.api,
                ineligibility_reason=reason,
                takes_default_engine=True,
            )
        tier.assert_not_awaited()
        assert mock_log.info.call_args.kwargs["billing_tier"] is None
        assert scoped_context.workflow_block_engine_override is None


@pytest.mark.asyncio
async def test_a_failing_billing_tier_lookup_cannot_decide_the_arm(scoped_context: SkyvernContext) -> None:
    # The resolver's catch-all turns any exception inside it into control, so a tier read that
    # escaped would let a billing-lookup outage move every run of the experiment onto v1 -- a
    # dependency the arm never had. The treated run must still be treated.
    provider = FakeExperimentationProvider({WORKFLOW_TASK_V3_AB_FLAG: True})
    with patch(WORKFLOW_BLOCK_ENGINE_APP_TARGET) as mock_app:
        mock_app.EXPERIMENTATION_PROVIDER = provider
        mock_app.AGENT_FUNCTION.resolve_billing_tier = AsyncMock(side_effect=RuntimeError("billing down"))
        await resolve_workflow_block_engine_arm(
            scoped_context,
            workflow_run_id="wr_tier_down",
            organization_id="org_1",
            workflow_permanent_id="wpid_1",
            workflow_status=WorkflowStatus.published,
            trigger_type=WorkflowRunTriggerType.api,
            ineligibility_reason=None,
            takes_default_engine=True,
        )

    assert scoped_context.workflow_block_engine_override == RunEngine.skyvern_v3
    by_flag = {flag: properties or {} for flag, _distinct_id, properties in provider.calls}
    assert by_flag[WORKFLOW_TASK_V3_AB_FLAG][BILLING_TIER_PROPERTY] == "unknown"


@pytest.mark.asyncio
async def test_new_self_serve_workflow_defaults_to_v3_without_consulting_the_ab_flag(
    scoped_context: SkyvernContext,
    v3_default_cutoff: datetime,
) -> None:
    # The whole point of the rule: the percentage knob is bypassed, so a flag sitting at 0% (or a
    # bucket that landed on control) cannot take a new self-serve workflow back to v1.
    provider = FakeExperimentationProvider({WORKFLOW_TASK_V3_AB_FLAG: False})
    run_id = "wr_new_self_serve_enrolled"
    resolution = await resolve_arm(
        scoped_context,
        provider,
        workflow_run_id=run_id,
        ineligibility_reason=None,
        billing_tier=BillingTier.SELF_SERVE,
        first_version_created_at=v3_default_cutoff.replace(tzinfo=None) + timedelta(seconds=1),
    )

    assert scoped_context.workflow_block_engine_override == RunEngine.skyvern_v3
    assert _make_block(TaskBlock, label="new_wf").resolve_engine(run_id) == RunEngine.skyvern_v3
    # Only the kill switch is read: no rollout flag and no percentage knob gate an eligible run.
    assert consulted_flags(provider) == [DISABLE_TASK_V3_FLAG]
    assert resolution.log["route_reason"] == WorkflowBlockEngineRouteReason.new_self_serve_workflow_default
    assert resolution.log["billing_tier"] == BillingTier.SELF_SERVE.value


@pytest.mark.asyncio
async def test_a_new_version_of_an_older_workflow_is_not_a_new_workflow(
    scoped_context: SkyvernContext, v3_default_cutoff: datetime
) -> None:
    # A workflow saved again after the cutoff keeps an old birthday. resolve_arm wires the latest
    # version at LATEST_VERSION_CREATED_AT, on the far side of the cutoff, so a resolver that read
    # the version this run executes would enrol this workflow and flip the arm below to treatment --
    # which is the regression that would otherwise take every long-lived workflow onto v3 at once.
    provider = FakeExperimentationProvider({WORKFLOW_TASK_V3_AB_FLAG: False})
    resolution = await resolve_arm(
        scoped_context,
        provider,
        workflow_run_id="wr_old_wf",
        ineligibility_reason=None,
        billing_tier=BillingTier.SELF_SERVE,
        first_version_created_at=v3_default_cutoff.replace(tzinfo=None) - timedelta(days=30),
    )

    assert scoped_context.workflow_block_engine_override is None
    assert DISABLE_TASK_V3_FLAG in [flag for flag, _distinct_id, _properties in provider.calls]
    assert WORKFLOW_TASK_V3_AB_FLAG in [flag for flag, _distinct_id, _properties in provider.calls]
    assert resolution.log["route_reason"] == WorkflowBlockEngineRouteReason.flag_bucket_control
    # Asserted here rather than inside the fake, whose AssertionError the rule's catch-all would
    # swallow into "not a new workflow": the birth timestamp is read once, scoped to this run's own
    # organization. The read itself is the min over every version of the id including deleted ones,
    # so a deleted or copilot-test first version cannot make a long-lived workflow look unborn.
    resolution.birth_reads.assert_awaited_once_with("wpid_1", "org_1")


@pytest.mark.asyncio
async def test_a_per_call_auto_generated_workflow_is_never_a_new_workflow(
    scoped_context: SkyvernContext, v3_default_cutoff: datetime
) -> None:
    # /v1/run/tasks/login, /v1/run/tasks/download_files, the two credential test-login endpoints and
    # the SDK run endpoint each mint a fresh workflow_permanent_id per call and run it while it is
    # still auto_generated, holding a v3-eligible block. Every one of those is born after any cutoff,
    # so without this exclusion setting the cutoff moves all of that standing API traffic onto v3 at
    # once instead of only the workflows customers keep.
    provider = FakeExperimentationProvider({WORKFLOW_TASK_V3_AB_FLAG: False})
    resolution = await resolve_arm(
        scoped_context,
        provider,
        workflow_run_id="wr_auto_generated",
        ineligibility_reason=None,
        billing_tier=BillingTier.SELF_SERVE,
        first_version_created_at=v3_default_cutoff.replace(tzinfo=None) + timedelta(days=1),
        workflow_status=WorkflowStatus.auto_generated,
    )

    assert scoped_context.workflow_block_engine_override is None
    assert WORKFLOW_TASK_V3_AB_FLAG in consulted_flags(provider)
    assert resolution.log["route_reason"] == WorkflowBlockEngineRouteReason.flag_bucket_control
    # A status the rule cannot use must not pay for the workflow read either.
    resolution.birth_reads.assert_not_awaited()


@pytest.mark.parametrize(
    "trigger_type",
    [WorkflowRunTriggerType.job_recipe_apply, WorkflowRunTriggerType.job_recipe_extract],
)
@pytest.mark.asyncio
async def test_a_per_call_recipe_run_is_never_a_new_workflow(
    scoped_context: SkyvernContext,
    v3_default_cutoff: datetime,
    trigger_type: WorkflowRunTriggerType,
) -> None:
    # The job-recipe endpoints also mint a fresh workflow_permanent_id per request and run it in that
    # same request, but the definition they build carries no status and so is born published: the
    # auto_generated exclusion cannot see it, and every one of those ids is born after any cutoff.
    # One run per permanent id also means an enrolled recipe call has no control run of the same
    # workflow to be matched against, so the trigger kind is what keeps this standing API traffic
    # randomized by the A/B instead of force-routed.
    provider = FakeExperimentationProvider({WORKFLOW_TASK_V3_AB_FLAG: False})
    resolution = await resolve_arm(
        scoped_context,
        provider,
        workflow_run_id=f"wr_{trigger_type.value}",
        ineligibility_reason=None,
        billing_tier=BillingTier.SELF_SERVE,
        first_version_created_at=v3_default_cutoff.replace(tzinfo=None) + timedelta(days=1),
        workflow_status=WorkflowStatus.published,
        trigger_type=trigger_type,
    )

    assert scoped_context.workflow_block_engine_override is None
    assert WORKFLOW_TASK_V3_AB_FLAG in consulted_flags(provider)
    assert resolution.log["route_reason"] == WorkflowBlockEngineRouteReason.flag_bucket_control
    # A trigger the rule cannot use must not pay for the workflow read either.
    resolution.birth_reads.assert_not_awaited()


@pytest.mark.asyncio
async def test_a_workflow_born_auto_generated_and_kept_by_the_customer_is_a_new_workflow(
    scoped_context: SkyvernContext, v3_default_cutoff: datetime
) -> None:
    # POST /workflows/create-from-prompt -- the prompt box, the product's main new-workflow path --
    # writes version 1 as auto_generated whenever publish_workflow is false, which is its default, and
    # then drops the customer into the editor on that same workflow_permanent_id; an editor save
    # writes the next version as published. Excluding on the BIRTH version's status would therefore
    # exclude every prompt-created workflow forever, including the ones a customer edits and re-runs
    # for months, and the rule's whole stated purpose would be defeated for the path that creates most
    # of its population. The exclusion is about the version this run executes, not about how the id
    # was born.
    provider = FakeExperimentationProvider({WORKFLOW_TASK_V3_AB_FLAG: False})
    resolution = await resolve_arm(
        scoped_context,
        provider,
        workflow_run_id="wr_prompt_created_kept",
        ineligibility_reason=None,
        billing_tier=BillingTier.SELF_SERVE,
        first_version_created_at=v3_default_cutoff.replace(tzinfo=None) + timedelta(days=1),
        first_version_status=WorkflowStatus.auto_generated,
        workflow_status=WorkflowStatus.published,
    )

    assert scoped_context.workflow_block_engine_override == RunEngine.skyvern_v3
    assert resolution.log["route_reason"] == WorkflowBlockEngineRouteReason.new_self_serve_workflow_default
    assert WORKFLOW_TASK_V3_AB_FLAG not in consulted_flags(provider)


@pytest.mark.parametrize(
    ("first_version_created_at", "expect_v3_default"),
    [
        (datetime(2026, 9, 14, 23, 59, 59), False),
        # Naive rows come from datetime.utcnow(): the cutoff is aware, so a resolver that compared
        # them directly would raise and lose every enrolment. At-or-after includes the instant itself.
        (datetime(2026, 9, 15, 0, 0, 0), True),
        (datetime(2026, 9, 15, 0, 0, 1), True),
        (datetime(2026, 9, 15, 0, 0, 1, tzinfo=UTC), True),
        # 2026-09-14 21:00 at UTC-4 is 2026-09-15 01:00 UTC, i.e. after the cutoff despite the date.
        (datetime(2026, 9, 14, 21, 0, 0, tzinfo=timezone(timedelta(hours=-4))), True),
    ],
)
@pytest.mark.asyncio
async def test_cutoff_compares_aware_and_naive_first_version_timestamps_in_utc(
    scoped_context: SkyvernContext,
    v3_default_cutoff: datetime,
    first_version_created_at: datetime,
    expect_v3_default: bool,
) -> None:
    provider = FakeExperimentationProvider({WORKFLOW_TASK_V3_AB_FLAG: False})
    resolution = await resolve_arm(
        scoped_context,
        provider,
        workflow_run_id=f"wr_cutoff_{first_version_created_at.isoformat()}",
        ineligibility_reason=None,
        billing_tier=BillingTier.SELF_SERVE,
        first_version_created_at=first_version_created_at,
    )

    expected_override = RunEngine.skyvern_v3 if expect_v3_default else None
    assert scoped_context.workflow_block_engine_override == expected_override
    expected_reason = (
        WorkflowBlockEngineRouteReason.new_self_serve_workflow_default
        if expect_v3_default
        else WorkflowBlockEngineRouteReason.flag_bucket_control
    )
    assert resolution.log["route_reason"] == expected_reason


@pytest.mark.skipif(not hasattr(time, "tzset"), reason="TZ cannot be changed in-process on this platform")
def test_a_naive_created_at_is_read_as_utc_not_as_local_time(monkeypatch: pytest.MonkeyPatch) -> None:
    # "Naive means UTC" and the classic astimezone() spelling that means "naive means local" are the
    # same function on a UTC machine, which CI and every container is -- so neither the cases above
    # nor any assertion made under the ambient zone can tell them apart. workflows.created_at is
    # datetime.utcnow(), so reading it as local time misclassifies every workflow born within the
    # process's TZ offset of the cutoff, silently enrolling or excluding runs that have no control arm.
    monkeypatch.setenv("TZ", "America/New_York")
    time.tzset()
    try:
        assert _as_utc(datetime(2026, 9, 15, 12)) == datetime(2026, 9, 15, 12, tzinfo=UTC)
    finally:
        monkeypatch.undo()
        time.tzset()


@pytest.mark.parametrize("billing_tier", [BillingTier.ENTERPRISE, BillingTier.UNKNOWN])
@pytest.mark.asyncio
async def test_only_the_self_serve_tier_gets_the_new_workflow_default(
    scoped_context: SkyvernContext, v3_default_cutoff: datetime, billing_tier: BillingTier
) -> None:
    # UNKNOWN is a failed lookup, so it must fall through with enterprise rather than be force-routed.
    provider = FakeExperimentationProvider({WORKFLOW_TASK_V3_AB_FLAG: True})
    resolution = await resolve_arm(
        scoped_context,
        provider,
        workflow_run_id=f"wr_tier_{billing_tier.value}",
        ineligibility_reason=None,
        billing_tier=billing_tier,
        first_version_created_at=v3_default_cutoff.replace(tzinfo=None) + timedelta(days=1),
    )

    assert WORKFLOW_TASK_V3_AB_FLAG in [flag for flag, _distinct_id, _properties in provider.calls]
    assert resolution.log["route_reason"] == WorkflowBlockEngineRouteReason.flag_bucket_treatment
    # A tier the rule cannot use must not pay for the workflow read either.
    resolution.birth_reads.assert_not_awaited()


@pytest.mark.asyncio
async def test_kill_switch_wins_over_the_new_workflow_default(
    scoped_context: SkyvernContext, v3_default_cutoff: datetime
) -> None:
    provider = FakeExperimentationProvider({WORKFLOW_TASK_V3_AB_FLAG: True, DISABLE_TASK_V3_FLAG: True})
    resolution = await resolve_arm(
        scoped_context,
        provider,
        workflow_run_id="wr_new_but_killed",
        ineligibility_reason=None,
        billing_tier=BillingTier.SELF_SERVE,
        first_version_created_at=v3_default_cutoff.replace(tzinfo=None) + timedelta(days=1),
    )

    assert scoped_context.workflow_block_engine_override is None
    assert resolution.log["route_reason"] == WorkflowBlockEngineRouteReason.disabled
    resolution.birth_reads.assert_not_awaited()


@pytest.mark.asyncio
async def test_no_cutoff_leaves_every_workflow_on_the_flag_path(scoped_context: SkyvernContext) -> None:
    # Deliberately does not request v3_default_cutoff: the autouse pin holds the setting at None.
    provider = FakeExperimentationProvider({WORKFLOW_TASK_V3_AB_FLAG: False})
    resolution = await resolve_arm(
        scoped_context,
        provider,
        workflow_run_id="wr_no_cutoff",
        ineligibility_reason=None,
        billing_tier=BillingTier.SELF_SERVE,
        first_version_created_at=datetime(2030, 1, 1),
    )

    assert scoped_context.workflow_block_engine_override is None
    assert resolution.log["route_reason"] == WorkflowBlockEngineRouteReason.flag_bucket_control
    resolution.birth_reads.assert_not_awaited()


@pytest.mark.asyncio
async def test_ineligible_run_is_never_defaulted_to_v3(
    scoped_context: SkyvernContext, v3_default_cutoff: datetime
) -> None:
    # Run-level eligibility is about whether the run can be rerouted at all, so the rule changes the
    # arm decision only: an unsupported or pinned block still executes as authored.
    provider = FakeExperimentationProvider({WORKFLOW_TASK_V3_AB_FLAG: True})
    resolution = await resolve_arm(
        scoped_context,
        provider,
        workflow_run_id="wr_new_but_ineligible",
        ineligibility_reason=V3AbIneligibleReason.unsupported_block,
        billing_tier=BillingTier.SELF_SERVE,
        first_version_created_at=v3_default_cutoff.replace(tzinfo=None) + timedelta(days=1),
    )

    assert scoped_context.workflow_block_engine_override is None
    assert provider.calls == []
    assert resolution.log["route_reason"] == WorkflowBlockEngineRouteReason.ineligible
    resolution.birth_reads.assert_not_awaited()


@pytest.mark.asyncio
async def test_a_failing_first_version_read_falls_back_to_the_flag_path(
    scoped_context: SkyvernContext, v3_default_cutoff: datetime
) -> None:
    # The new rule must not be able to take the run off the arm it would have had: a database blip
    # falls back to the A/B, not to control, and never forces v3.
    provider = FakeExperimentationProvider({WORKFLOW_TASK_V3_AB_FLAG: True})
    resolution = await resolve_arm(
        scoped_context,
        provider,
        workflow_run_id="wr_first_version_down",
        ineligibility_reason=None,
        billing_tier=BillingTier.SELF_SERVE,
        first_version_error=RuntimeError("workflows read failed"),
    )

    assert scoped_context.workflow_block_engine_override == RunEngine.skyvern_v3
    assert resolution.log["route_reason"] == WorkflowBlockEngineRouteReason.flag_bucket_treatment
    # The fallback is silent in the arm it produces, so the swallowed read has to be visible
    # somewhere: a rule that stops enrolling because a query broke must not look like a rule nobody
    # set the cutoff for.
    assert [call.kwargs.get("workflow_permanent_id") for call in resolution.warnings] == ["wpid_1"]


@pytest.mark.asyncio
async def test_a_missing_first_version_falls_back_to_the_flag_path(
    scoped_context: SkyvernContext, v3_default_cutoff: datetime
) -> None:
    provider = FakeExperimentationProvider({WORKFLOW_TASK_V3_AB_FLAG: False})
    resolution = await resolve_arm(
        scoped_context,
        provider,
        workflow_run_id="wr_first_version_missing",
        ineligibility_reason=None,
        billing_tier=BillingTier.SELF_SERVE,
        first_version_created_at=None,
    )

    assert scoped_context.workflow_block_engine_override is None
    assert resolution.log["route_reason"] == WorkflowBlockEngineRouteReason.flag_bucket_control


CHOSEN_ENGINE_CUTOFF = datetime(2026, 10, 1, tzinfo=UTC)


async def _resolve_chosen_engine_run(
    context: SkyvernContext,
    monkeypatch: pytest.MonkeyPatch,
    blocks: list[BaseTaskBlock],
    *,
    run_id: str,
    cutoff: datetime | None,
    born_at: datetime,
    flags: dict[str, bool] | None = None,
    workflow_status: WorkflowStatus = WorkflowStatus.published,
    values: dict[str, str] | None = None,
    strict_error_flags: set[str] | None = None,
) -> tuple[FakeExperimentationProvider, Any]:
    # Eligibility and the unset-engine fact are computed from the blocks by the same helpers the
    # workflow service calls, so a block-level regression in either reaches these assertions.
    monkeypatch.setattr(settings, "TASK_V3_CHOSEN_ENGINE_CUTOFF", cutoff)
    provider = FakeExperimentationProvider(
        flags if flags is not None else {WORKFLOW_TASK_V3_AB_FLAG: True},
        values=values,
        strict_error_flags=strict_error_flags,
    )
    resolution = await resolve_arm(
        context,
        provider,
        workflow_run_id=run_id,
        ineligibility_reason=v3_ab_ineligibility_reason(blocks),
        takes_default_engine=takes_default_engine(blocks),
        billing_tier=BillingTier.ENTERPRISE,
        first_version_created_at=born_at,
        workflow_status=workflow_status,
    )
    return provider, resolution


_AFTER = CHOSEN_ENGINE_CUTOFF.replace(tzinfo=None) + timedelta(seconds=1)
_BEFORE = CHOSEN_ENGINE_CUTOFF.replace(tzinfo=None) - timedelta(days=30)
_R = WorkflowBlockEngineRouteReason


@pytest.mark.parametrize(
    ("cutoff", "born_at", "chosen", "expected_engine", "expected_reason"),
    [
        # Cutoff unset or workflow born before it: today's routing. The A/B is at 100% treatment, so an
        # unset or skyvern_v1 block is rerouted and a v2/v3 pin makes the run ineligible.
        (None, _AFTER, None, RunEngine.skyvern_v3, _R.flag_bucket_treatment),
        (None, _AFTER, RunEngine.skyvern_v1, RunEngine.skyvern_v3, _R.flag_bucket_treatment),
        (None, _AFTER, RunEngine.skyvern_v2, RunEngine.skyvern_v2, _R.ineligible),
        (None, _AFTER, RunEngine.skyvern_v3, RunEngine.skyvern_v3, _R.ineligible),
        (CHOSEN_ENGINE_CUTOFF, _BEFORE, None, RunEngine.skyvern_v3, _R.flag_bucket_treatment),
        (CHOSEN_ENGINE_CUTOFF, _BEFORE, RunEngine.skyvern_v1, RunEngine.skyvern_v3, _R.flag_bucket_treatment),
        (CHOSEN_ENGINE_CUTOFF, _BEFORE, RunEngine.skyvern_v2, RunEngine.skyvern_v2, _R.ineligible),
        (CHOSEN_ENGINE_CUTOFF, _BEFORE, RunEngine.skyvern_v3, RunEngine.skyvern_v3, _R.ineligible),
        # Born at or after the cutoff: an unset engine runs on v3 and a chosen one runs as written.
        (CHOSEN_ENGINE_CUTOFF, _AFTER, None, RunEngine.skyvern_v3, _R.new_workflow_v3_default),
        (CHOSEN_ENGINE_CUTOFF, _AFTER, RunEngine.skyvern_v1, RunEngine.skyvern_v1, _R.chosen_engine),
        (CHOSEN_ENGINE_CUTOFF, _AFTER, RunEngine.skyvern_v2, RunEngine.skyvern_v2, _R.chosen_engine),
        (CHOSEN_ENGINE_CUTOFF, _AFTER, RunEngine.skyvern_v3, RunEngine.skyvern_v3, _R.chosen_engine),
    ],
)
@pytest.mark.asyncio
async def test_chosen_engine_cutoff_matrix(
    scoped_context: SkyvernContext,
    monkeypatch: pytest.MonkeyPatch,
    cutoff: datetime | None,
    born_at: datetime,
    chosen: RunEngine | None,
    expected_engine: RunEngine,
    expected_reason: WorkflowBlockEngineRouteReason,
) -> None:
    block = _make_block(TaskBlock, label="only", engine=chosen)
    provider, resolution = await _resolve_chosen_engine_run(
        scoped_context, monkeypatch, [block], run_id="wr_matrix", cutoff=cutoff, born_at=born_at
    )

    assert block.resolve_engine("wr_matrix") == expected_engine
    assert resolution.log["route_reason"] == expected_reason
    if expected_reason in (_R.new_workflow_v3_default, _R.chosen_engine):
        # Every tier, with no A/B draw and no billing-tier lookup; only an unset block asks the new-workflow split.
        new_split = [WORKFLOW_TASK_V3_AB_NEW_WORKFLOWS_FLAG] if expected_reason == _R.new_workflow_v3_default else []
        assert consulted_flags(provider) == [DISABLE_TASK_V3_FLAG, *new_split]
        assert resolution.log["billing_tier"] is None


@pytest.mark.asyncio
async def test_a_post_cutoff_run_honors_each_block_on_its_own(
    scoped_context: SkyvernContext, monkeypatch: pytest.MonkeyPatch
) -> None:
    # The A/B keeps a run single-engine; a chosen-engine run does not, so a v3 pin (which makes the run
    # A/B-ineligible) must not hold the unset blocks on v1, and a block v3 cannot run stays on v1.
    unset = _make_block(NavigationBlock, label="unset", navigation_goal="Apply to the job")
    chose_v1 = _make_block(TaskBlock, label="chose_v1", engine=RunEngine.skyvern_v1)
    chose_v3 = _make_block(TaskBlock, label="chose_v3", engine=RunEngine.skyvern_v3)
    unsupported = _make_block(ValidationBlock, label="unsupported", complete_on_download=True)
    blocks: list[BaseTaskBlock] = [unset, chose_v1, chose_v3, unsupported]
    _provider, resolution = await _resolve_chosen_engine_run(
        scoped_context, monkeypatch, blocks, run_id="wr_mixed", cutoff=CHOSEN_ENGINE_CUTOFF, born_at=_AFTER
    )

    assert [block.resolve_engine("wr_mixed") for block in blocks] == [
        RunEngine.skyvern_v3,
        RunEngine.skyvern_v1,
        RunEngine.skyvern_v3,
        RunEngine.skyvern_v1,
    ]
    assert resolution.log["route_reason"] == _R.new_workflow_v3_default


@pytest.mark.parametrize(
    ("values", "strict_error_flags", "expected_engine", "expected_reason"),
    [
        ({}, None, RunEngine.skyvern_v3, _R.new_workflow_v3_default),
        ({"WORKFLOW_TASK_V3_AB_NEW_WORKFLOWS": "v3"}, None, RunEngine.skyvern_v3, _R.new_workflow_ab_treatment),
        ({"WORKFLOW_TASK_V3_AB_NEW_WORKFLOWS": "v1"}, None, RunEngine.skyvern_v1, _R.new_workflow_ab_control),
        ({"WORKFLOW_TASK_V3_AB_NEW_WORKFLOWS": "50"}, None, RunEngine.skyvern_v3, _R.new_workflow_v3_default),
        ({}, {"WORKFLOW_TASK_V3_AB_NEW_WORKFLOWS"}, RunEngine.skyvern_v3, _R.new_workflow_v3_default),
    ],
    ids=["not-listed", "treatment", "control", "unknown-variant", "raised-read"],
)
@pytest.mark.asyncio
async def test_the_new_workflow_split_moves_only_unset_blocks(
    scoped_context: SkyvernContext,
    monkeypatch: pytest.MonkeyPatch,
    values: dict[str, str],
    strict_error_flags: set[str] | None,
    expected_engine: RunEngine,
    expected_reason: WorkflowBlockEngineRouteReason,
) -> None:
    # Only a listed run's control variant moves its unset blocks to v1; a v3 pin stays v3 in every cell, and anything
    # that is not a listed variant keeps today's v3 default.
    unset = _make_block(NavigationBlock, label="unset", navigation_goal="Apply to the job")
    chose_v3 = _make_block(TaskBlock, label="chose_v3", engine=RunEngine.skyvern_v3)
    provider, resolution = await _resolve_chosen_engine_run(
        scoped_context,
        monkeypatch,
        [unset, chose_v3],
        run_id="wr_new_split",
        cutoff=CHOSEN_ENGINE_CUTOFF,
        born_at=_AFTER,
        values=values,
        strict_error_flags=strict_error_flags,
    )

    assert [unset.resolve_engine("wr_new_split"), chose_v3.resolve_engine("wr_new_split")] == [
        expected_engine,
        RunEngine.skyvern_v3,
    ]
    assert resolution.log["route_reason"] == expected_reason
    assert WORKFLOW_TASK_V3_AB_FLAG not in consulted_flags(provider)
    (call,) = (c for c in provider.calls if c[0] == WORKFLOW_TASK_V3_AB_NEW_WORKFLOWS_FLAG)
    # Both properties are what a release condition can list; a segment listed by organization never matches without it.
    assert call[1:] == ("wr_new_split", {"organization_id": "org_1", "workflow_permanent_id": "wpid_1"})


@pytest.mark.asyncio
async def test_the_new_workflow_split_never_reaches_a_pre_cutoff_or_killed_run(
    scoped_context: SkyvernContext, monkeypatch: pytest.MonkeyPatch
) -> None:
    control = {"WORKFLOW_TASK_V3_AB_NEW_WORKFLOWS": "v1"}
    before = _make_block(TaskBlock, label="before")
    provider, resolution = await _resolve_chosen_engine_run(
        scoped_context,
        monkeypatch,
        [before],
        run_id="wr_pre",
        cutoff=CHOSEN_ENGINE_CUTOFF,
        born_at=_BEFORE,
        values=control,
    )
    assert before.resolve_engine("wr_pre") == RunEngine.skyvern_v3
    assert resolution.log["route_reason"] == _R.flag_bucket_treatment
    assert WORKFLOW_TASK_V3_AB_NEW_WORKFLOWS_FLAG not in consulted_flags(provider)

    killed = _make_block(TaskBlock, label="killed")
    provider, resolution = await _resolve_chosen_engine_run(
        scoped_context,
        monkeypatch,
        [killed],
        run_id="wr_killed_split",
        cutoff=CHOSEN_ENGINE_CUTOFF,
        born_at=_AFTER,
        flags={WORKFLOW_TASK_V3_AB_FLAG: True, DISABLE_TASK_V3_FLAG: True},
        values={"WORKFLOW_TASK_V3_AB_NEW_WORKFLOWS": "v3"},
    )
    assert killed.resolve_engine("wr_killed_split") == RunEngine.skyvern_v1
    assert resolution.log["route_reason"] == _R.disabled
    assert WORKFLOW_TASK_V3_AB_NEW_WORKFLOWS_FLAG not in consulted_flags(provider)


@pytest.mark.parametrize("chosen", [None, RunEngine.skyvern_v3])
@pytest.mark.asyncio
async def test_kill_switch_wins_over_the_chosen_engine_cutoff(
    scoped_context: SkyvernContext, monkeypatch: pytest.MonkeyPatch, chosen: RunEngine | None
) -> None:
    block = _make_block(TaskBlock, label="only", engine=chosen)
    _provider, resolution = await _resolve_chosen_engine_run(
        scoped_context,
        monkeypatch,
        [block],
        run_id="wr_killed",
        cutoff=CHOSEN_ENGINE_CUTOFF,
        born_at=_AFTER,
        flags={WORKFLOW_TASK_V3_AB_FLAG: True, DISABLE_TASK_V3_FLAG: True},
    )

    assert scoped_context.workflow_block_engine_override is None
    resolution.birth_reads.assert_not_awaited()
    if chosen is None:
        assert block.resolve_engine("wr_killed") == RunEngine.skyvern_v1
        assert resolution.log["route_reason"] == _R.disabled
    else:
        # The pin is recorded as written and the dispatch gate stops it; its label predates this rule.
        assert block.resolve_engine("wr_killed") == RunEngine.skyvern_v3
        assert resolution.log["route_reason"] == _R.ineligible


@pytest.mark.asyncio
async def test_a_per_call_workflow_born_after_the_chosen_engine_cutoff_keeps_ab_routing(
    scoped_context: SkyvernContext, monkeypatch: pytest.MonkeyPatch
) -> None:
    # Every per-call platform workflow is born after any cutoff; enrolling them would move standing
    # API traffic, so their explicit skyvern_v1 stays routable.
    block = _make_block(TaskBlock, label="only", engine=RunEngine.skyvern_v1)
    _provider, resolution = await _resolve_chosen_engine_run(
        scoped_context,
        monkeypatch,
        [block],
        run_id="wr_per_call",
        cutoff=CHOSEN_ENGINE_CUTOFF,
        born_at=_AFTER,
        workflow_status=WorkflowStatus.auto_generated,
    )

    assert block.resolve_engine("wr_per_call") == RunEngine.skyvern_v3
    assert resolution.log["route_reason"] == _R.flag_bucket_treatment
    resolution.birth_reads.assert_not_awaited()


def test_an_omitted_engine_is_unset_and_a_stored_one_is_kept() -> None:
    omitted = NavigationBlockYAML(label="nav", url="https://example.com", navigation_goal="Go")
    stored = NavigationBlockYAML.model_validate(
        {"label": "nav", "url": "https://example.com", "navigation_goal": "Go", "engine": "skyvern-1.0"}
    )

    assert omitted.engine is None
    assert stored.engine == RunEngine.skyvern_v1
    block = _make_block(NavigationBlock, label="nav", navigation_goal="Go")
    assert block.engine is None
    assert type(block).model_validate(block.model_dump()).engine is None
    # Unset routes like skyvern_v1 before the cutoff, so it never reads as a pin.
    assert v3_ab_ineligibility_reason([block]) is None
    assert takes_default_engine([block]) is True


def test_an_unset_engine_is_left_out_of_the_stored_definition_so_an_older_image_reads_it() -> None:
    # The field as the image before SKY-17436 declares it: a stored null fails validation there, so a
    # pod still on it during a rollout, or after a revert, could not load the workflow.
    class OlderImageTaskBlock(BaseModel):
        model_config = ConfigDict(extra="allow")
        engine: RunEngine = RunEngine.skyvern_v1

    unset = _make_block(NavigationBlock, label="unset", navigation_goal="Go")
    chosen = _make_block(TaskBlock, label="chosen", engine=RunEngine.skyvern_v3)
    stored = WorkflowDefinition(parameters=[], blocks=[unset, chosen]).model_dump(mode="json")

    assert "engine" not in stored["blocks"][0]
    assert [OlderImageTaskBlock.model_validate(b).engine for b in stored["blocks"]] == [
        RunEngine.skyvern_v1,
        RunEngine.skyvern_v3,
    ]
    reloaded = WorkflowDefinition.model_validate(stored)
    assert [b.engine for b in reloaded.blocks] == [None, RunEngine.skyvern_v3]
    with pytest.raises(ValidationError):
        OlderImageTaskBlock.model_validate({"engine": None})


@pytest.mark.asyncio
async def test_both_cutoff_rules_share_one_birth_read(
    scoped_context: SkyvernContext, monkeypatch: pytest.MonkeyPatch, v3_default_cutoff: datetime
) -> None:
    # Born between the two cutoffs: the chosen-engine rule reads the birth and declines, then the
    # self-serve rule needs the same timestamp.
    monkeypatch.setattr(settings, "TASK_V3_CHOSEN_ENGINE_CUTOFF", CHOSEN_ENGINE_CUTOFF)
    resolution = await resolve_arm(
        scoped_context,
        FakeExperimentationProvider(),
        workflow_run_id="wr_between",
        ineligibility_reason=None,
        billing_tier=BillingTier.SELF_SERVE,
        first_version_created_at=CHOSEN_ENGINE_CUTOFF - timedelta(days=1),
    )

    assert resolution.log["route_reason"] == _R.new_self_serve_workflow_default
    assert resolution.birth_reads.await_count == 1


@pytest.mark.parametrize(
    ("cutoff", "born_at", "expected"),
    [
        (None, _AFTER, None),
        (CHOSEN_ENGINE_CUTOFF, _BEFORE, None),
        (CHOSEN_ENGINE_CUTOFF, _AFTER, RunEngine.skyvern_v3),
    ],
)
@pytest.mark.asyncio
async def test_effective_default_engine_follows_the_chosen_engine_cutoff(
    monkeypatch: pytest.MonkeyPatch, cutoff: datetime | None, born_at: datetime, expected: RunEngine | None
) -> None:
    monkeypatch.setattr(settings, "TASK_V3_CHOSEN_ENGINE_CUTOFF", cutoff)
    with patch(WORKFLOW_BLOCK_ENGINE_APP_TARGET) as mock_app:
        mock_app.EXPERIMENTATION_PROVIDER = FakeExperimentationProvider()
        mock_app.DATABASE.workflows.get_workflow_permanent_id_created_at = AsyncMock(return_value=born_at)
        assert await effective_default_engine("wpid_1", "org_1") == expected
        # With no flag provider the resolver never applies the rule, so the editor must not claim it.
        mock_app.EXPERIMENTATION_PROVIDER = NoOpExperimentationProvider()
        assert await effective_default_engine("wpid_1", "org_1") is None


@pytest.mark.parametrize(
    ("engine", "pinned", "born_at", "expected_engine", "expected_reason", "expected_pins"),
    [
        # An older workflow's unmarked skyvern-1.0 is usually the old editor's Default and stays in the A/B.
        (RunEngine.skyvern_v1, False, _BEFORE, RunEngine.skyvern_v3, _R.flag_bucket_treatment, 0),
        (RunEngine.skyvern_v1, True, _BEFORE, RunEngine.skyvern_v1, _R.pinned_v1_engine, 1),
        (None, False, _BEFORE, RunEngine.skyvern_v3, _R.flag_bucket_treatment, 0),
        (RunEngine.skyvern_v3, False, _BEFORE, RunEngine.skyvern_v3, _R.ineligible, 0),
        (RunEngine.skyvern_v1, True, _AFTER, RunEngine.skyvern_v1, _R.chosen_engine, 1),
    ],
)
@pytest.mark.asyncio
async def test_only_a_pinned_skyvern_v1_leaves_the_ab_on_an_older_workflow(
    scoped_context: SkyvernContext,
    monkeypatch: pytest.MonkeyPatch,
    engine: RunEngine | None,
    pinned: bool,
    born_at: datetime,
    expected_engine: RunEngine,
    expected_reason: WorkflowBlockEngineRouteReason,
    expected_pins: int,
) -> None:
    monkeypatch.setattr(settings, "TASK_V3_CHOSEN_ENGINE_CUTOFF", CHOSEN_ENGINE_CUTOFF)
    block = _make_block(TaskBlock, label="e2e_block", engine=engine, engine_pinned=pinned)
    with patch("skyvern.forge.sdk.experimentation.workflow_block_engine.LOG") as mock_log:
        persisted = await _persisted_engine_from_execute_workflow_blocks(
            monkeypatch,
            FakeExperimentationProvider({WORKFLOW_TASK_V3_AB_FLAG: True}),
            first_version_created_at=born_at,
            block=block,
        )

    assert persisted == expected_engine
    log = mock_log.info.call_args.kwargs
    assert log["route_reason"] == expected_reason
    assert log.get("pinned_v1_blocks", 0) == expected_pins


@pytest.mark.parametrize(
    "block",
    [
        {"block_type": "navigation", "navigation_goal": "Go"},
        {"block_type": "task", "navigation_goal": "Go"},
        {"block_type": "action", "navigation_goal": "Go"},
        {"block_type": "extraction", "data_extraction_goal": "Read"},
        {"block_type": "login", "navigation_goal": "Go"},
        {"block_type": "file_download", "navigation_goal": "Go"},
        {"block_type": "validation", "complete_criterion": "Done"},
    ],
    ids=lambda block: block["block_type"],
)
def test_a_pinned_skyvern_v1_survives_the_stored_definition_and_counts_as_a_change(block: dict[str, str]) -> None:
    def definition(pinned: bool) -> Any:
        nav = {**block, "label": "nav", "url": "https://example.com"}
        yaml = WorkflowDefinitionYAML.model_validate(
            {"parameters": [], "blocks": [{**nav, "engine": "skyvern-1.0", "engine_pinned": pinned}]}
        )
        return convert_workflow_definition(yaml, "w_test")

    pinned, unmarked = definition(True), definition(False)
    stored = pinned.model_dump(mode="json")["blocks"][0]

    assert stored["engine_pinned"] is True
    assert "engine_pinned" not in unmarked.model_dump(mode="json")["blocks"][0]
    assert WorkflowDefinition.model_validate(pinned.model_dump(mode="json")).blocks[0].engine_pinned is True
    # Before the cutoff an unmarked skyvern-1.0 and an unset engine compare alike; a pin must not.
    assert workflow_definitions_differ(unmarked, pinned)


_PIN_ADDED_EVENT = "Workflow version adds a skyvern-1.0 engine pin"
_WAREHOUSE_PIN_PATTERN = '"engine":"skyvern-1.0","engine_pinned":true'


def _nav(label: str, engine: str | None = None, pinned: bool = False) -> dict[str, Any]:
    block: dict[str, Any] = {
        "block_type": "navigation",
        "label": label,
        "url": "https://e.com",
        "navigation_goal": "Go",
    }
    if engine is not None:
        block["engine"] = engine
    if pinned:
        block["engine_pinned"] = True
    return block


def _v1_pin(label: str) -> dict[str, Any]:
    return _nav(label, "skyvern-1.0", pinned=True)


def _for_loop(*inner: dict[str, Any]) -> dict[str, Any]:
    return {"block_type": "for_loop", "label": "loop", "loop_variable_reference": "{{ items }}", "loop_blocks": inner}


def _definition(blocks: list[dict[str, Any]]) -> WorkflowDefinition:
    return convert_workflow_definition(
        WorkflowDefinitionYAML.model_validate({"parameters": [], "blocks": blocks}), "w_test"
    )


def _workflow_row(blocks: list[dict[str, Any]], version: int, created_by: str | None = None) -> Workflow:
    now = datetime.now(UTC)
    return Workflow(
        workflow_id=f"w_v{version}",
        organization_id="o_test",
        workflow_permanent_id="wpid_test",
        title="test",
        version=version,
        is_saved_task=False,
        workflow_definition=_definition(blocks),
        created_at=now,
        modified_at=now,
        created_by=created_by,
    )


async def _save_definition(
    monkeypatch: pytest.MonkeyPatch,
    new_blocks: list[dict[str, Any]],
    *,
    overwritten: Workflow | None,
    get_previous_version: AsyncMock,
    created_by: str | None = None,
    get_overwritten: AsyncMock | None = None,
    dialect: str | None = "postgresql",
) -> tuple[Workflow, Workflow, MagicMock]:
    saved = _workflow_row(new_blocks, version=3, created_by=created_by)
    engine = SimpleNamespace() if dialect is None else SimpleNamespace(dialect=SimpleNamespace(name=dialect))
    monkeypatch.setattr(app.DATABASE, "engine", engine)
    monkeypatch.setattr(app.DATABASE.workflows, "get_workflow", get_overwritten or AsyncMock(return_value=overwritten))
    monkeypatch.setattr(app.DATABASE.workflows, "get_workflow_by_permanent_id", get_previous_version)
    monkeypatch.setattr(
        app.DATABASE.workflows, "update_workflow_and_reconcile_definition_params", AsyncMock(return_value=saved)
    )
    monkeypatch.setattr(app.DATABASE.organizations, "get_organization", AsyncMock(return_value=None))
    monkeypatch.setattr(app.DATABASE.workflow_runs, "count_recent_workflow_runs", AsyncMock(return_value=327))
    monkeypatch.setattr("skyvern.forge.sdk.workflow.service.encrypt_workflow_definition_secrets", AsyncMock())
    with patch("skyvern.forge.sdk.workflow.service.LOG") as mock_log:
        returned = await WorkflowService().update_workflow_definition(
            workflow_id=saved.workflow_id,
            organization_id="o_test",
            workflow_definition=saved.workflow_definition,
            notify_workflow_saved=False,
        )
    return returned, saved, mock_log


def _pin_added_logs(mock_log: MagicMock) -> list[dict[str, Any]]:
    return [call.kwargs for call in mock_log.info.call_args_list if call.args and call.args[0] == _PIN_ADDED_EVENT]


@pytest.mark.parametrize(
    ("previous_blocks", "overwritten_blocks", "new_blocks", "created_by", "expected"),
    [
        pytest.param([_nav("a", "skyvern-1.0")], [], [_v1_pin("a")], None, (0, 1, 2), id="new_version_adds_pin"),
        pytest.param([_v1_pin("a")], [], [_v1_pin("a")], None, None, id="new_version_keeps_pin"),
        pytest.param([_v1_pin("a"), _v1_pin("b")], [], [_v1_pin("a")], None, None, id="new_version_unpins"),
        pytest.param(
            [_v1_pin("a"), _nav("b")], [], [_nav("b"), _v1_pin("a")], None, None, id="new_version_reorders_pin"
        ),
        pytest.param(
            [_v1_pin("a")], [], [_v1_pin("a"), _for_loop(_v1_pin("inner"))], None, (1, 2, 2), id="nested_second_pin"
        ),
        pytest.param(None, [], [_v1_pin("a")], None, (0, 1, None), id="first_version_with_pin"),
        pytest.param([_nav("a")], [_v1_pin("a")], [_v1_pin("a")], None, None, id="in_place_edit_keeps_pin"),
        pytest.param([_nav("a")], [], [_v1_pin("a")], COPILOT_TEST_WORKFLOW_CREATOR, None, id="copilot_test_row"),
    ],
)
@pytest.mark.asyncio
async def test_a_saved_version_logs_only_when_it_adds_a_skyvern_v1_pin(
    monkeypatch: pytest.MonkeyPatch,
    previous_blocks: list[dict[str, Any]] | None,
    overwritten_blocks: list[dict[str, Any]],
    new_blocks: list[dict[str, Any]],
    created_by: str | None,
    expected: tuple[int, int, int | None] | None,
) -> None:
    previous = None if previous_blocks is None else _workflow_row(previous_blocks, version=2)
    returned, saved, mock_log = await _save_definition(
        monkeypatch,
        new_blocks,
        overwritten=_workflow_row(overwritten_blocks, version=3),
        get_previous_version=AsyncMock(return_value=previous),
        created_by=created_by,
    )

    assert returned is saved
    logs = _pin_added_logs(mock_log)
    if expected is None:
        assert logs == []
        return
    before, after, previous_version = expected
    [log] = logs
    assert log["workflow_permanent_id"] == "wpid_test"
    assert log["version"] == 3
    assert log["previous_version"] == previous_version
    assert log["pinned_v1_blocks_before"] == before
    assert log["pinned_v1_blocks_after"] == after
    assert log["recent_runs_7d"] == 327


@pytest.mark.parametrize("idempotent", [pytest.param(True, id="idempotent_create"), pytest.param(False, id="create")])
@pytest.mark.asyncio
async def test_a_first_version_created_with_a_v1_pin_logs(
    monkeypatch: pytest.MonkeyPatch, tmp_path: Path, idempotent: bool
) -> None:
    database = AgentDB(f"sqlite+aiosqlite:///{tmp_path / 'pin.db'}")
    async with database.engine.begin() as connection:
        await connection.run_sync(Base.metadata.create_all)
    organization = await database.organizations.create_organization(organization_name="Test", organization_id="o_test")
    monkeypatch.setattr(app, "DATABASE", database)
    monkeypatch.setattr(app, "WORKFLOW_SERVICE", WorkflowService())
    # A file database has its own connection per task, unlike the in-memory one the dialect gate exists for.
    monkeypatch.setattr(
        WorkflowService, "_pinned_v1_blocks_to_check", staticmethod(lambda d: pinned_v1_block_count(d.blocks))
    )
    request = WorkflowCreateYAMLRequest.model_validate(
        {"title": "test", "workflow_definition": {"parameters": [], "blocks": [_v1_pin("a")]}}
    )
    try:
        with patch("skyvern.forge.sdk.workflow.service.LOG") as mock_log:
            saved = await app.WORKFLOW_SERVICE.create_workflow_from_request(
                organization=organization,
                request=request,
                new_workflow_permanent_id="wpid_test" if idempotent else None,
            )
    finally:
        await database.engine.dispose()

    [log] = _pin_added_logs(mock_log)
    assert (log["workflow_permanent_id"], log["version"]) == (saved.workflow_permanent_id, 1)
    assert (log["pinned_v1_blocks_before"], log["pinned_v1_blocks_after"]) == (0, 1)


@pytest.mark.parametrize(
    ("blocks", "dialect"),
    [
        # Without a v1 pin the save must not touch the engine; some test doubles of the database have none.
        pytest.param([_nav("a", "skyvern-1.0"), _nav("b", "skyvern-2.0", pinned=True)], None, id="no_v1_pin"),
        # An in-memory SQLite database pools one connection, so a read on another task would roll the save back.
        pytest.param([_v1_pin("a")], "sqlite", id="v1_pin_on_sqlite"),
    ],
)
@pytest.mark.asyncio
async def test_a_save_reads_no_prior_row_without_a_v1_pin_or_on_sqlite(
    monkeypatch: pytest.MonkeyPatch, blocks: list[dict[str, Any]], dialect: str | None
) -> None:
    reads: list[dict[str, Any]] = []

    async def prior_row_read(**kwargs: Any) -> None:
        reads.append(kwargs)

    returned, saved, mock_log = await _save_definition(
        monkeypatch,
        blocks,
        overwritten=None,
        get_previous_version=AsyncMock(side_effect=prior_row_read),
        get_overwritten=AsyncMock(side_effect=prior_row_read),
        dialect=dialect,
    )

    assert returned is saved
    assert reads == []
    assert _pin_added_logs(mock_log) == []


@pytest.mark.asyncio
async def test_a_failing_pin_read_never_runs_on_the_save_task_or_fails_the_save(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    save_task = asyncio.current_task()
    read_tasks: list[asyncio.Task[Any] | None] = []

    async def failing_read(**_: Any) -> Workflow:
        read_tasks.append(asyncio.current_task())
        raise RuntimeError("db down")

    returned, saved, mock_log = await _save_definition(
        monkeypatch,
        [_v1_pin("a")],
        overwritten=None,
        get_previous_version=AsyncMock(side_effect=failing_read),
        get_overwritten=AsyncMock(side_effect=failing_read),
    )

    assert returned is saved
    assert len(read_tasks) == 2 and save_task not in read_tasks
    assert _pin_added_logs(mock_log) == []
    assert any(call.kwargs.get("workflow_permanent_id") == "wpid_test" for call in mock_log.warning.call_args_list)


@pytest.mark.parametrize(
    ("blocks", "expected_pins"),
    [
        pytest.param([_v1_pin("a")], 1, id="pinned_v1"),
        pytest.param([_nav("a", "skyvern-1.0")], 0, id="unpinned_v1"),
        pytest.param([_nav("a", "skyvern-3.0", pinned=True)], 0, id="pinned_v3"),
        pytest.param([_for_loop(_v1_pin("inner"))], 1, id="pinned_v1_in_for_loop"),
        pytest.param([_v1_pin("a"), _nav("b"), _v1_pin("c")], 2, id="two_pins"),
    ],
)
def test_the_warehouse_pin_regex_counts_what_the_service_counts(
    blocks: list[dict[str, Any]], expected_pins: int
) -> None:
    definition = _definition(blocks)
    stored = json.dumps(definition.model_dump(mode="json"), separators=(",", ":"), sort_keys=True)

    assert pinned_v1_block_count(get_all_blocks(definition.blocks)) == expected_pins
    assert stored.count(_WAREHOUSE_PIN_PATTERN) == expected_pins


@pytest.mark.parametrize(
    ("flags", "expected_reason", "displaced"),
    [
        ({WORKFLOW_TASK_V3_AB_FLAG: True}, WorkflowBlockEngineRouteReason.code_bucket_treatment, True),
        ({WORKFLOW_TASK_V3_AB_FLAG: False}, WorkflowBlockEngineRouteReason.code_bucket_control, False),
        ({}, WorkflowBlockEngineRouteReason.flag_undefined, False),
    ],
)
@pytest.mark.asyncio
async def test_a_code_mode_run_takes_the_ab_coin_and_only_treatment_leaves_its_script(
    scoped_context: SkyvernContext,
    flags: dict[str, bool],
    expected_reason: WorkflowBlockEngineRouteReason,
    displaced: bool,
) -> None:
    provider = FakeExperimentationProvider(flags)

    resolution = await resolve_arm(
        scoped_context, provider, workflow_run_id="wr_code", ineligibility_reason=None, would_be_code=True
    )

    assert resolution.log["route_reason"] == expected_reason
    assert resolution.log["would_be_code"] is True
    assert scoped_context.workflow_block_engine_arm_decision.would_be_code is True
    assert code_mode_displaced("wr_code") is displaced
    assert code_mode_displaced("wr_other_run") is False
    # Sent for reads only; no release condition may target it (see _ab_flag_puts_run_in_treatment).
    [ab_properties] = [props for flag, _, props in provider.calls if flag == WORKFLOW_TASK_V3_AB_FLAG]
    assert ab_properties[WOULD_BE_CODE_PROPERTY] == "true"


@pytest.mark.asyncio
async def test_the_new_workflow_defaults_never_take_a_code_mode_run_off_its_script(
    scoped_context: SkyvernContext, monkeypatch: pytest.MonkeyPatch, v3_default_cutoff: datetime
) -> None:
    born_after = v3_default_cutoff.replace(tzinfo=None) + timedelta(days=1)
    self_serve = await resolve_arm(
        scoped_context,
        FakeExperimentationProvider({WORKFLOW_TASK_V3_AB_FLAG: False}),
        workflow_run_id="wr_self_serve_code",
        ineligibility_reason=None,
        billing_tier=BillingTier.SELF_SERVE,
        first_version_created_at=born_after,
        would_be_code=True,
    )
    assert self_serve.log["route_reason"] == WorkflowBlockEngineRouteReason.code_bucket_control

    monkeypatch.setattr(settings, "TASK_V3_CHOSEN_ENGINE_CUTOFF", v3_default_cutoff)
    chosen = await resolve_arm(
        scoped_context,
        FakeExperimentationProvider({WORKFLOW_TASK_V3_AB_FLAG: True}),
        workflow_run_id="wr_chosen_code",
        ineligibility_reason=None,
        first_version_created_at=born_after,
        would_be_code=True,
    )
    # The cutoff rule keeps routing a script run's uncached blocks and AI fallback, as before.
    assert chosen.log["route_reason"] == WorkflowBlockEngineRouteReason.new_workflow_v3_default
    assert code_mode_displaced("wr_self_serve_code") is False
    assert code_mode_displaced("wr_chosen_code") is False


@pytest.mark.asyncio
async def test_a_new_workflow_code_mode_run_on_control_falls_back_to_v1_and_keeps_its_script(
    scoped_context: SkyvernContext, monkeypatch: pytest.MonkeyPatch
) -> None:
    # A script run's uncached blocks and AI fallback resolve through the block, so they follow the split's arm.
    monkeypatch.setattr(settings, "TASK_V3_CHOSEN_ENGINE_CUTOFF", CHOSEN_ENGINE_CUTOFF)
    resolution = await resolve_arm(
        scoped_context,
        FakeExperimentationProvider(
            {WORKFLOW_TASK_V3_AB_FLAG: True}, values={WORKFLOW_TASK_V3_AB_NEW_WORKFLOWS_FLAG: "v1"}
        ),
        workflow_run_id="wr_new_code_control",
        ineligibility_reason=None,
        first_version_created_at=_AFTER,
        would_be_code=True,
    )
    assert resolution.log["route_reason"] == _R.new_workflow_ab_control
    assert _make_block(TaskBlock, label="uncached").resolve_engine("wr_new_code_control") == RunEngine.skyvern_v1
    assert code_mode_displaced("wr_new_code_control") is False


@pytest.mark.asyncio
async def test_a_treatment_code_mode_run_executes_on_v3_without_loading_or_minting_a_script(
    scoped_context: SkyvernContext, monkeypatch: pytest.MonkeyPatch
) -> None:
    script = MagicMock(script_id="s_cached", script_revision_id="sr_cached")
    load_blocks = AsyncMock(return_value=[])
    monkeypatch.setattr(app.DATABASE.scripts, "get_script_blocks_by_script_revision_id", load_blocks)
    scoped_context.generate_script = True  # what a code_gen run carries into execution

    engine = await _persisted_engine_from_execute_workflow_blocks(
        monkeypatch, FakeExperimentationProvider({WORKFLOW_TASK_V3_AB_FLAG: True}), script=script
    )

    assert engine == RunEngine.skyvern_v3
    load_blocks.assert_not_awaited()
    assert scoped_context.script_id is None
    assert scoped_context.generate_script is False


@pytest.mark.parametrize("born_after_chosen_engine_cutoff", [False, True])
@pytest.mark.asyncio
async def test_an_explicit_code_generation_request_stays_on_v1(
    scoped_context: SkyvernContext, monkeypatch: pytest.MonkeyPatch, born_after_chosen_engine_cutoff: bool
) -> None:
    # A run asked to produce a script must not land on v3, whose actions yield no usable one: neither by
    # the A/B nor by the chosen-engine rule for new workflows.
    born_at = None
    if born_after_chosen_engine_cutoff:
        monkeypatch.setattr(settings, "TASK_V3_CHOSEN_ENGINE_CUTOFF", CHOSEN_ENGINE_CUTOFF)
        born_at = CHOSEN_ENGINE_CUTOFF.replace(tzinfo=None) + timedelta(days=1)
    engine = await _persisted_engine_from_execute_workflow_blocks(
        monkeypatch,
        FakeExperimentationProvider({WORKFLOW_TASK_V3_AB_FLAG: True}),
        code_gen=True,
        first_version_created_at=born_at,
    )

    assert engine == RunEngine.skyvern_v1


def _download_context(tmp_path: Path, *, holding: bool = False) -> MagicMock:
    interceptor = CDPDownloadInterceptor(
        output_dir=str(tmp_path), network_egress_monitor=MagicMock(), redirect_hop_authorizer=AsyncMock()
    )
    interceptor.set_replay_candidate_mode(holding)
    context = MagicMock()
    context._skyvern_cdp_download_interceptor = interceptor
    return context


def _holding(context: MagicMock) -> bool:
    return context._skyvern_cdp_download_interceptor._replay_candidate_mode


def _v3_download_block(block_cls: type[BaseTaskBlock], **overrides: Any) -> BaseTaskBlock:
    fields: dict[str, Any] = {
        "label": "export",
        "output_parameter": output_parameter(),
        "url": "https://site.example/export",
        "complete_on_download": True,
        "engine": RunEngine.skyvern_v3,
        **overrides,
    }
    return block_cls(**fields)


class _TaskRan(Exception):
    """Raised by the stubbed execute_step: what follows it is not under test here."""


async def _execute_block(block: BaseTaskBlock) -> None:
    skyvern_context.set(SkyvernContext(organization_id="o_test", workflow_run_id="wr_task_block_test"))
    try:
        with contextlib.suppress(_TaskRan):
            await block.execute(
                workflow_run_id="wr_task_block_test", workflow_run_block_id="wrb_test", organization_id="o_test"
            )
    finally:
        skyvern_context.reset()


@contextlib.contextmanager
def _v3_block_deps(working_page_url: str, context: MagicMock | None = None) -> Iterator[Any]:
    with (
        mock_block_execute_deps(working_page_url=working_page_url) as deps,
        patch("skyvern.forge.sdk.workflow.models.block.task_v3_disabled", AsyncMock(return_value=False)),
    ):
        deps.task.organization_id = "o_test"
        if context is not None:
            deps.browser_state.browser_context = context
        yield deps


@pytest.mark.parametrize("cancelled_at", ["profile_settle", "screenshot"])
@pytest.mark.asyncio
async def test_a_block_cancelled_after_its_start_url_ends_the_hold_it_began(tmp_path: Path, cancelled_at: str) -> None:
    block = _v3_download_block(FileDownloadBlock)
    context = _download_context(tmp_path)
    with _v3_block_deps("about:blank", context) as deps:
        page = deps.browser_state.must_get_working_page.return_value
        if cancelled_at == "profile_settle":
            deps.workflow_run.browser_profile_id = "bp_test"
            deps.browser_state.browser_artifacts.applied_browser_profile_id = "bp_test"
            page.wait_for_load_state = AsyncMock(side_effect=asyncio.CancelledError())
        else:
            deps.browser_state.take_fullpage_screenshot = AsyncMock(side_effect=asyncio.CancelledError())
        with pytest.raises(asyncio.CancelledError):
            await _execute_block(block)

        deps.browser_state.navigate_to_url.assert_awaited_once()
        assert _holding(context) is False


@pytest.mark.asyncio
async def test_a_first_block_holds_before_the_browser_navigates_and_ends_only_the_holds_it_began(
    tmp_path: Path,
) -> None:
    block = _v3_download_block(TaskBlock, title="Export")
    replaced, current = _download_context(tmp_path / "a"), _download_context(tmp_path / "b")
    borrowed = _download_context(tmp_path / "c", holding=True)
    holding_during_task: dict[str, bool] = {}
    with _v3_block_deps("https://site.example/export", current) as deps:

        async def get_or_create_for_workflow_run(**kwargs: Any) -> Any:
            before_navigation = kwargs.get("before_navigation")
            for context in (replaced, current, borrowed):
                if before_navigation is not None:
                    before_navigation(SimpleNamespace(browser_context=context))
            return deps.browser_state

        async def execute_step(**_kwargs: Any) -> None:
            holding_during_task.update(replaced=_holding(replaced), current=_holding(current))
            raise _TaskRan

        deps.browser_manager.get_or_create_for_workflow_run = AsyncMock(side_effect=get_or_create_for_workflow_run)
        deps.agent.execute_step = AsyncMock(side_effect=execute_step)
        await _execute_block(block)

    assert holding_during_task == {"replaced": True, "current": True}
    assert (_holding(replaced), _holding(current), _holding(borrowed)) == (False, False, True)


@pytest.mark.parametrize(
    "overrides", [{"complete_on_download": False}, {"engine": RunEngine.skyvern_v1}], ids=["not-download", "v1"]
)
@pytest.mark.asyncio
async def test_a_block_that_does_not_hold_leaves_browser_creation_unchanged(
    tmp_path: Path, overrides: dict[str, Any]
) -> None:
    block = _v3_download_block(TaskBlock, title="Export", **overrides)
    context = _download_context(tmp_path)
    with _v3_block_deps("https://site.example/export", context) as deps:
        deps.agent.execute_step = AsyncMock(side_effect=_TaskRan)
        await _execute_block(block)

        assert deps.browser_manager.get_or_create_for_workflow_run.await_args.kwargs["before_navigation"] is None
        assert _holding(context) is False


@pytest.mark.asyncio
async def test_a_later_block_still_holds_through_its_own_navigation(tmp_path: Path) -> None:
    block = _v3_download_block(TaskBlock, title="Export")
    context = _download_context(tmp_path)
    holding_at_navigation: list[bool] = []
    with (
        _v3_block_deps("https://site.example/previous", context) as deps,
        patch.object(TaskBlock, "get_task_order", AsyncMock(return_value=(1, 0))),
    ):
        deps.browser_manager.get_for_workflow_run = MagicMock(return_value=deps.browser_state)
        deps.browser_state.navigate_to_url = AsyncMock(
            side_effect=lambda **_: holding_at_navigation.append(_holding(context))
        )
        deps.agent.execute_step = AsyncMock(side_effect=_TaskRan)
        await _execute_block(block)

    assert holding_at_navigation == [True]
    assert _holding(context) is False


@pytest.mark.asyncio
async def test_a_block_keeps_a_replacement_context_its_task_watch_held_through_cleanup(tmp_path: Path) -> None:
    block = _v3_download_block(TaskBlock, title="Export")
    replacement = _download_context(tmp_path)
    replacement.cookies = AsyncMock(return_value=[])
    interceptor = replacement._skyvern_cdp_download_interceptor
    interceptor._browser_context = replacement
    page = MagicMock()
    page.context = replacement
    seen: dict[str, Any] = {}
    reply = MagicMock(body=b"account,amount\nA-1,12.50\n", content_type="text/csv", filename="export.csv", status=200)

    async def _page() -> Any:
        return page

    async def execute_step(**_kwargs: Any) -> None:
        watch = DownloadFailureWatch(_page)
        await watch.attach()
        watch.close()
        with patch.object(cdp_module.file_api, "fetch_file_bytes", AsyncMock(return_value=reply), create=True):
            await interceptor._download_url_directly("https://site.example/export.csv", "export.csv")
        seen.update(held=len(interceptor.replay_candidates()), landed=sorted(p.name for p in tmp_path.iterdir()))
        raise _TaskRan

    with _v3_block_deps("https://site.example/export") as deps:
        deps.agent.execute_step = AsyncMock(side_effect=execute_step)
        await _execute_block(block)

    assert seen == {"held": 1, "landed": []}
    assert _holding(replacement) is False


@pytest.mark.asyncio
async def test_parallel_and_nested_blocks_each_end_only_their_own_holds(tmp_path: Path) -> None:
    outer, sibling, nested = (_download_context(tmp_path / name) for name in ("outer", "sibling", "nested"))
    blocks = {
        name: _v3_download_block(TaskBlock, label=name, title=name, complete_on_download=False)
        for name in ("outer", "sibling", "nested")
    }
    outer_holding, sibling_done = asyncio.Event(), asyncio.Event()
    observed: dict[str, bool] = {}

    async def execute_step(*, task_block: BaseTaskBlock, **_kwargs: Any) -> None:
        if task_block.label == "outer":
            begin_replay_candidate_hold(outer)
            outer_holding.set()
            await sibling_done.wait()
            await _execute_block(blocks["nested"])
            observed["outer_after_sibling_and_nested"] = _holding(outer)
        elif task_block.label == "sibling":
            await outer_holding.wait()
            begin_replay_candidate_hold(sibling)
        else:
            begin_replay_candidate_hold(nested)
        raise _TaskRan

    async def sibling_then_signal() -> None:
        try:
            await _execute_block(blocks["sibling"])
        finally:
            sibling_done.set()

    with mock_block_execute_deps(working_page_url="https://site.example/export") as deps:
        deps.task.organization_id = "o_test"
        deps.agent.execute_step = AsyncMock(side_effect=execute_step)
        await asyncio.gather(_execute_block(blocks["outer"]), sibling_then_signal())

    assert observed == {"outer_after_sibling_and_nested": True}
    assert (_holding(outer), _holding(sibling), _holding(nested)) == (False, False, False)


@pytest.mark.asyncio
async def test_a_replay_a_failed_earlier_block_admitted_saves_as_before_inside_the_next_v3_blocks_hold(
    tmp_path: Path,
) -> None:
    # An earlier block's navigation fails with its export in flight; execute_safe lets the next V3 download block hold.
    earlier = TaskBlock(
        label="earlier",
        output_parameter=output_parameter(),
        title="Earlier",
        url="https://site.example/export",
        continue_on_failure=True,
    )
    later = _v3_download_block(TaskBlock, label="later", title="Later")
    context = _download_context(tmp_path)
    context.cookies = AsyncMock(return_value=[])
    interceptor = context._skyvern_cdp_download_interceptor
    interceptor._browser_context = context
    released = asyncio.Event()
    csv_bytes = b"account,amount\nA-1,12.50\n"
    in_flight: list[asyncio.Task[None]] = []
    holding_when_it_landed: list[bool] = []

    async def slow_fetch(*_args: Any, **_kwargs: Any) -> Any:
        await released.wait()
        return MagicMock(body=csv_bytes, content_type="text/csv", filename="export.csv", status=200)

    async def earlier_navigation_fails(**_kwargs: Any) -> Any:
        in_flight.append(
            asyncio.create_task(interceptor._download_url_directly("https://site.example/export.csv", "export.csv"))
        )
        await asyncio.sleep(0)
        raise FailedToNavigateToUrl(url="https://site.example/export", error_message="net::ERR_ABORTED")

    @contextlib.asynccontextmanager
    async def running_run(*_args: Any) -> Any:
        yield SimpleNamespace(status=WorkflowRunStatus.running)

    async def task_order_by_label(block: TaskBlock, *_args: Any) -> tuple[int, int]:
        return (0, 0) if block.label == "earlier" else (1, 0)

    async def later_navigation(**_kwargs: Any) -> None:
        holding_when_it_landed.append(_holding(context))
        released.set()
        await in_flight[0]

    with (
        _v3_block_deps("https://site.example/previous", context) as deps,
        patch.object(TaskBlock, "get_task_order", task_order_by_label),
        patch.object(cdp_module.file_api, "fetch_file_bytes", slow_fetch, create=True),
    ):
        deps.browser_manager.get_or_create_for_workflow_run = AsyncMock(side_effect=earlier_navigation_fails)
        deps.browser_manager.get_for_workflow_run = MagicMock(return_value=deps.browser_state)
        deps.browser_state.navigate_to_url = AsyncMock(side_effect=later_navigation)
        deps.agent.execute_step = AsyncMock(side_effect=_TaskRan)
        deps.observer_db.create_workflow_run_block = AsyncMock(return_value=MagicMock(workflow_run_block_id="wrb_test"))
        block_module.app.DATABASE.workflow_runs.create_or_update_workflow_run_output_parameter = AsyncMock()
        block_module.app.DATABASE.workflow_runs.admit_workflow_run_block_dispatch = running_run
        skyvern_context.set(SkyvernContext(organization_id="o_test", workflow_run_id="wr_task_block_test"))
        try:
            result = await earlier.execute_safe(workflow_run_id="wr_task_block_test", organization_id="o_test")
        finally:
            skyvern_context.reset()
        await _execute_block(later)

    assert result.status == BlockStatus.failed and result.can_continue_after_failure
    assert holding_when_it_landed == [True]
    assert interceptor.replay_candidates() == {}
    assert (tmp_path / "export.csv").read_bytes() == csv_bytes