Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
Log a closed-set ineligibility reason when resolving the workflow-block engine arm (#8335)
pedrohsdb22b28ca20e54
diff --git a/skyvern/forge/sdk/experimentation/workflow_block_engine.py b/skyvern/forge/sdk/experimentation/workflow_block_engine.pyindex c6cf29fe4..0a6c3541a 100644--- a/skyvern/forge/sdk/experimentation/workflow_block_engine.py+++ b/skyvern/forge/sdk/experimentation/workflow_block_engine.py@@ -1,5 +1,7 @@from __future__ import annotations+from typing import TYPE_CHECKING+import structlogfrom skyvern.forge import app@@ -7,6 +9,10 @@ from skyvern.forge.sdk.core import skyvern_contextfrom skyvern.forge.sdk.experimentation.providers import NoOpExperimentationProviderfrom skyvern.schemas.run_enums import RunEngine+if TYPE_CHECKING:+ # Runtime import would cycle: block.py imports workflow_block_engine_override from this module.+ from skyvern.forge.sdk.workflow.models.block import V3AbIneligibleReason+LOG = structlog.get_logger()WORKFLOW_TASK_V3_AB_FLAG = "WORKFLOW_TASK_V3_AB"@@ -30,7 +36,7 @@ async def resolve_workflow_block_engine_arm(workflow_run_id: str,organization_id: str | None,workflow_permanent_id: str | None,- run_is_eligible: bool,+ ineligibility_reason: V3AbIneligibleReason | None,) -> None:"""Resolve the workflow-block engine A/B once at execution start and pin the arm on the context.@@ -53,6 +59,7 @@ async def resolve_workflow_block_engine_arm(if context.workflow_block_engine_resolved_run_id == workflow_run_id:returnoverride: RunEngine | None = None+ run_is_eligible = ineligibility_reason is Nonetry:if run_is_eligible:# The kill switch, shared with the dispatch gate via task_v3_disabled so both@@ -84,6 +91,7 @@ async def resolve_workflow_block_engine_arm(workflow_permanent_id=workflow_permanent_id,arm="treatment" if override else "control",run_is_eligible=run_is_eligible,+ ineligibility_reason=ineligibility_reason,)diff --git a/skyvern/forge/sdk/workflow/models/block.py b/skyvern/forge/sdk/workflow/models/block.pyindex da8b7811c..694bead03 100644--- a/skyvern/forge/sdk/workflow/models/block.py+++ b/skyvern/forge/sdk/workflow/models/block.py@@ -28,6 +28,7 @@ from collections.abc import Sequencefrom dataclasses import replacefrom datetime import UTC, date, datetime, timefrom email.message import EmailMessage+from enum import StrEnumfrom functools import partialfrom pathlib import Path, PurePosixPathfrom time import monotonic@@ -15283,8 +15284,23 @@ def _task_block_supports_v3(task_block: BaseTaskBlock) -> bool:return task_block.block_type in _TASK_V3_SUPPORTED_BLOCK_TYPES-def run_is_eligible_for_v3_ab(blocks: list[BlockTypeVar], *, is_script_run: bool) -> bool:- """Whether a whole workflow run may be rerouted onto v3 by the A/B.+class V3AbIneligibleReason(StrEnum):+ """The closed set of reasons a run is excluded from the workflow-block engine A/B.++ Parity with the bare-task route reason (``RunRouteReason`` in ``cloud/agent_functions.py``):+ a closed set an operator can slice logs by, instead of a bare boolean that answers "is+ eligible" but not "why not."+ """++ script_run = "script_run"+ pinned_engine = "pinned_engine"+ unsupported_block = "unsupported_block"+ block_totp_verification_url = "block_totp_verification_url"+ no_reroutable_blocks = "no_reroutable_blocks"+++def v3_ab_ineligibility_reason(blocks: list[BlockTypeVar], *, is_script_run: bool) -> V3AbIneligibleReason | None:+ """Why a whole workflow run may not be rerouted onto v3 by the A/B, or None if it may.Eligibility is a property of the RUN, not of a block: a run whose blocks disagreed about theengine would drive one browser session with two engines, so no per-run outcome would be@@ -15305,7 +15321,7 @@ def run_is_eligible_for_v3_ab(blocks: list[BlockTypeVar], *, is_script_run: boolexecution."""if is_script_run:- return False+ return V3AbIneligibleReason.script_runreroutable_blocks = 0for block in blocks:if not isinstance(block, BaseTaskBlock):@@ -15313,18 +15329,25 @@ def run_is_eligible_for_v3_ab(blocks: list[BlockTypeVar], *, is_script_run: boolif block.block_type in _ENGINE_INERT_BLOCK_TYPES:continueif block.engine != RunEngine.skyvern_v1:- return False+ return V3AbIneligibleReason.pinned_engineif not _task_block_supports_v3(block):- return False+ return V3AbIneligibleReason.unsupported_block# A/B rerouting never admits a run with a verification-URL block (a block explicitly pinned to# v3 is honored as requested); bare tasks with a verification URL are rerouted. Block-run# code/budget dynamics are unmeasured (SKY-14816).if block.totp_verification_url:- return False+ return V3AbIneligibleReason.block_totp_verification_urlreroutable_blocks += 1# A run with nothing to reroute would be bucketed and recorded as an exposure while both arms# execute identically, diluting the experiment.- return reroutable_blocks > 0+ if reroutable_blocks == 0:+ return V3AbIneligibleReason.no_reroutable_blocks+ return None+++def run_is_eligible_for_v3_ab(blocks: list[BlockTypeVar], *, is_script_run: bool) -> bool:+ """Whether a whole workflow run may be rerouted onto v3 by the A/B; see v3_ab_ineligibility_reason."""+ return v3_ab_ineligibility_reason(blocks, is_script_run=is_script_run) is Nonedef get_all_blocks(blocks: list[BlockTypeVar]) -> list[BlockTypeVar]:diff --git a/skyvern/forge/sdk/workflow/service.py b/skyvern/forge/sdk/workflow/service.pyindex 3538ad137..37e29c9a8 100644--- a/skyvern/forge/sdk/workflow/service.py+++ b/skyvern/forge/sdk/workflow/service.py@@ -151,7 +151,7 @@ from skyvern.forge.sdk.workflow.models.block import (compute_conditional_scopes,get_all_blocks,resolve_conditional_merge_edges,- run_is_eligible_for_v3_ab,+ v3_ab_ineligibility_reason,)from skyvern.forge.sdk.workflow.models.parameter import (AWSSecretParameter,@@ -5583,7 +5583,7 @@ class WorkflowService:is_script_run = await self.should_run_script(workflow, workflow_run)# Resolve the workflow-block engine A/B once, before any block runs: eligibility is a- # property of the whole run (see run_is_eligible_for_v3_ab), and every block of a run must+ # property of the whole run (see v3_ab_ineligibility_reason), and every block of a run must# share an arm. Covers the DAG executor too, which this method delegates to.current_context = skyvern_context.current()if current_context:@@ -5592,7 +5592,7 @@ class WorkflowService:workflow_run_id=workflow_run_id,organization_id=organization_id,workflow_permanent_id=workflow_run.workflow_permanent_id,- run_is_eligible=run_is_eligible_for_v3_ab(all_blocks, is_script_run=is_script_run),+ ineligibility_reason=v3_ab_ineligibility_reason(all_blocks, is_script_run=is_script_run),)else:LOG.warning(diff --git a/tests/unit/test_workflow_block_engine.py b/tests/unit/test_workflow_block_engine.pyindex 0829408ca..49df64aaa 100644--- a/tests/unit/test_workflow_block_engine.py+++ b/tests/unit/test_workflow_block_engine.py@@ -5,7 +5,7 @@ workflow_run_blocks and the dispatched engine come from the same resolution.from __future__ import annotations-from collections.abc import Iterator+from collections.abc import Callable, Iteratorfrom datetime import UTC, datetimefrom typing import Anyfrom unittest.mock import AsyncMock, MagicMock, patch@@ -33,12 +33,15 @@ from skyvern.forge.sdk.workflow.models.block import (NavigationBlock,TaskBlock,UrlBlock,+ V3AbIneligibleReason,ValidationBlock,get_all_blocks,run_is_eligible_for_v3_ab,+ v3_ab_ineligibility_reason,)from skyvern.forge.sdk.workflow.service import WorkflowServicefrom skyvern.schemas.run_enums import RunEngine+from skyvern.schemas.workflows import BlockTypefrom tests.unit.helpers import make_organizationfrom tests.unit.test_agent_task_v3 import _make_block, _make_output_parameter, _run_execute_step_gatefrom tests.unit.test_block_description_caching import _block_result, _setup_mocks@@ -82,7 +85,7 @@ async def _resolve(provider: BaseExperimentationProvider,*,workflow_run_id: str,- run_is_eligible: bool,+ ineligibility_reason: V3AbIneligibleReason | None,organization_id: str | None = "org_1",workflow_permanent_id: str | None = "wpid_1",) -> None:@@ -93,7 +96,7 @@ async def _resolve(workflow_run_id=workflow_run_id,organization_id=organization_id,workflow_permanent_id=workflow_permanent_id,- run_is_eligible=run_is_eligible,+ ineligibility_reason=ineligibility_reason,)@@ -138,7 +141,12 @@ async def test_mixed_eligibility_run_pins_whole_run_to_control(scoped_context: Sassert run_is_eligible_for_v3_ab(blocks, is_script_run=False) is Falseprovider = _FakeExperimentationProvider({WORKFLOW_TASK_V3_AB_FLAG: True})- await _resolve(scoped_context, provider, workflow_run_id="wr_mixed", run_is_eligible=False)+ await _resolve(+ scoped_context,+ provider,+ workflow_run_id="wr_mixed",+ ineligibility_reason=V3AbIneligibleReason.block_totp_verification_url,+ )for block in blocks:assert block.resolve_engine("wr_mixed") == RunEngine.skyvern_v1@@ -149,7 +157,7 @@ async def test_mixed_eligibility_run_pins_whole_run_to_control(scoped_context: S@pytest.mark.asyncioasync def test_explicit_block_engine_is_never_overridden_by_treatment_arm(scoped_context: SkyvernContext) -> None:provider = _FakeExperimentationProvider({WORKFLOW_TASK_V3_AB_FLAG: True})- await _resolve(scoped_context, provider, workflow_run_id="wr_pinned", run_is_eligible=True)+ await _resolve(scoped_context, provider, workflow_run_id="wr_pinned", ineligibility_reason=None)assert scoped_context.workflow_block_engine_override == RunEngine.skyvern_v3cua_block = _make_block(TaskBlock, label="cua", engine=RunEngine.openai_cua)@@ -180,7 +188,7 @@ async def test_all_eligible_run_resolves_every_block_to_treatment(scoped_contextassert run_is_eligible_for_v3_ab(blocks, is_script_run=False) is Trueprovider = _FakeExperimentationProvider({WORKFLOW_TASK_V3_AB_FLAG: True})- await _resolve(scoped_context, provider, workflow_run_id="wr_treatment", run_is_eligible=True)+ await _resolve(scoped_context, provider, workflow_run_id="wr_treatment", ineligibility_reason=None)for block in blocks:assert block.resolve_engine("wr_treatment") == RunEngine.skyvern_v3@@ -189,7 +197,7 @@ async def test_all_eligible_run_resolves_every_block_to_treatment(scoped_context@pytest.mark.asyncioasync def test_arm_resolved_once_per_run_survives_mid_run_flag_flip(scoped_context: SkyvernContext) -> None:provider = _FakeExperimentationProvider({WORKFLOW_TASK_V3_AB_FLAG: True})- await _resolve(scoped_context, provider, workflow_run_id="wr_once", run_is_eligible=True)+ await _resolve(scoped_context, provider, workflow_run_id="wr_once", ineligibility_reason=None)assert scoped_context.workflow_block_engine_override == RunEngine.skyvern_v3# Invalidate the provider's own 300s cache and flip the flag, so a second query would@@ -197,7 +205,7 @@ async def test_arm_resolved_once_per_run_survives_mid_run_flag_flip(scoped_conte# short-circuit on context.workflow_block_engine_resolved_run_id before that happens.provider.invalidate_resolution_caches()provider.flags[WORKFLOW_TASK_V3_AB_FLAG] = False- await _resolve(scoped_context, provider, workflow_run_id="wr_once", run_is_eligible=True)+ await _resolve(scoped_context, provider, workflow_run_id="wr_once", ineligibility_reason=None)assert scoped_context.workflow_block_engine_override == RunEngine.skyvern_v3@@ -207,11 +215,11 @@ async def test_different_run_id_on_same_context_reresolves_instead_of_inheritingscoped_context: SkyvernContext,) -> None:provider = _FakeExperimentationProvider({WORKFLOW_TASK_V3_AB_FLAG: True})- await _resolve(scoped_context, provider, workflow_run_id="wr_A", run_is_eligible=True)+ await _resolve(scoped_context, provider, workflow_run_id="wr_A", ineligibility_reason=None)assert workflow_block_engine_override("wr_A") == RunEngine.skyvern_v3provider.flags[WORKFLOW_TASK_V3_AB_FLAG] = False- await _resolve(scoped_context, provider, workflow_run_id="wr_B", run_is_eligible=True)+ await _resolve(scoped_context, provider, workflow_run_id="wr_B", ineligibility_reason=None)assert workflow_block_engine_override("wr_B") is None# The pin moved to B: A must not read as still-treatment via a stale resolution.@@ -221,7 +229,7 @@ async def test_different_run_id_on_same_context_reresolves_instead_of_inheriting@pytest.mark.asyncioasync def test_unresolved_run_id_reads_control_without_resolving(scoped_context: SkyvernContext) -> None:provider = _FakeExperimentationProvider({WORKFLOW_TASK_V3_AB_FLAG: True})- await _resolve(scoped_context, provider, workflow_run_id="wr_A", run_is_eligible=True)+ await _resolve(scoped_context, provider, workflow_run_id="wr_A", ineligibility_reason=None)assert workflow_block_engine_override("wr_A") == RunEngine.skyvern_v3# wr_B was never resolved (task_v2 / cached-script helper paths never call the resolver for@@ -256,7 +264,7 @@ async def test_resolver_matches_execute_step_flag_contract(scoped_context: Skyvescoped_context,resolver_provider,workflow_run_id="wr_contract",- run_is_eligible=True,+ ineligibility_reason=None,organization_id=(gate_call[2] or {}).get("organization_id"),workflow_permanent_id="wpid_contract",)@@ -273,7 +281,7 @@ async def test_resolver_matches_execute_step_flag_contract(scoped_context: Skyve@pytest.mark.asyncioasync def test_disable_flag_wins_over_ab_flag(scoped_context: SkyvernContext) -> None:provider = _FakeExperimentationProvider({WORKFLOW_TASK_V3_AB_FLAG: True, DISABLE_TASK_V3_FLAG: True})- await _resolve(scoped_context, provider, workflow_run_id="wr_disabled", run_is_eligible=True)+ await _resolve(scoped_context, provider, workflow_run_id="wr_disabled", ineligibility_reason=None)assert scoped_context.workflow_block_engine_override is Noneblock = _make_block(TaskBlock, label="disabled_block")@@ -283,7 +291,7 @@ async def test_disable_flag_wins_over_ab_flag(scoped_context: SkyvernContext) ->@pytest.mark.asyncioasync def test_provider_exception_fails_closed_to_control(scoped_context: SkyvernContext) -> None:provider = _FakeExperimentationProvider(raise_error=True)- await _resolve(scoped_context, provider, workflow_run_id="wr_err", run_is_eligible=True)+ await _resolve(scoped_context, provider, workflow_run_id="wr_err", ineligibility_reason=None)assert scoped_context.workflow_block_engine_override is Noneblock = _make_block(TaskBlock, label="err_block")@@ -309,7 +317,7 @@ async def test_noop_provider_never_queried_and_leaves_engine_unchanged(scoped_coworkflow_run_id="wr_noop",organization_id="org_1",workflow_permanent_id="wpid_1",- run_is_eligible=True,+ ineligibility_reason=None,)spy.assert_not_called()@@ -360,7 +368,7 @@ async def test_inert_blocks_resolve_to_v1_in_a_treated_run(scoped_context: Skyve# Eligibility skips GOTO_URL/HumanInteraction as engine-inert, so resolve_engine must skip# them too -- otherwise their workflow_run_blocks rows claim an engine that never ran.provider = _FakeExperimentationProvider({WORKFLOW_TASK_V3_AB_FLAG: True})- await _resolve(scoped_context, provider, workflow_run_id="wr_inert", run_is_eligible=True)+ await _resolve(scoped_context, provider, workflow_run_id="wr_inert", ineligibility_reason=None)assert scoped_context.workflow_block_engine_override == RunEngine.skyvern_v3url_block = _make_block(UrlBlock, label="goto", url="https://example.com")@@ -373,7 +381,7 @@ async def test_inert_blocks_resolve_to_v1_in_a_treated_run(scoped_context: Skyve@pytest.mark.asyncioasync def test_exclude_from_engine_ab_block_is_never_rerouted(scoped_context: SkyvernContext) -> None:provider = _FakeExperimentationProvider({WORKFLOW_TASK_V3_AB_FLAG: True})- await _resolve(scoped_context, provider, workflow_run_id="wr_excluded", run_is_eligible=True)+ await _resolve(scoped_context, provider, workflow_run_id="wr_excluded", ineligibility_reason=None)assert scoped_context.workflow_block_engine_override == RunEngine.skyvern_v3excluded_block = _make_block(ActionBlock, label="excluded")@@ -501,3 +509,100 @@ async def test_execute_workflow_blocks_pins_the_context_execute_safe_reads_from()assert captured["engine"] == RunEngine.skyvern_v3+++def _eligible_mix_blocks() -> list[BaseTaskBlock]:+ return [+ _make_block(TaskBlock, label="t1"),+ _make_block(NavigationBlock, label="t2", navigation_goal="Apply to the job"),+ _make_block(ActionBlock, label="t3"),+ ]+++def _pinned_engine_blocks() -> list[BaseTaskBlock]:+ eligible = _make_block(TaskBlock, label="ok")+ pinned = _make_block(+ NavigationBlock, label="pinned", navigation_goal="Apply to the job", engine=RunEngine.skyvern_v2+ )+ return [eligible, pinned]+++def _unsupported_block_type_blocks() -> list[BaseTaskBlock]:+ # No shipped BaseTaskBlock subclass declares an unsupported, non-inert block_type today, so+ # force one to exercise the branch _task_block_supports_v3 exists to guard.+ unsupported = _make_block(TaskBlock, label="unsupported")+ unsupported.block_type = BlockType.WAIT+ return [unsupported]+++def _download_gated_validation_blocks() -> list[BaseTaskBlock]:+ return [_make_block(ValidationBlock, label="dlv", complete_on_download=True)]+++def _totp_blocks() -> list[BaseTaskBlock]:+ return [_make_block(TaskBlock, label="totp", totp_verification_url="https://example.com/otp")]+++def _no_reroutable_blocks() -> list[BaseTaskBlock]:+ code_block = CodeBlock(label="code", output_parameter=_make_output_parameter("code"), code="pass")+ url_block = _make_block(UrlBlock, label="goto", url="https://example.com")+ human_block = _make_block(HumanInteractionBlock, label="human")+ return [code_block, url_block, human_block]+++ ("blocks_factory", "is_script_run", "expected_reason"),+ [+ (_eligible_mix_blocks, True, V3AbIneligibleReason.script_run),+ (_pinned_engine_blocks, False, V3AbIneligibleReason.pinned_engine),+ (_unsupported_block_type_blocks, False, V3AbIneligibleReason.unsupported_block),+ (_download_gated_validation_blocks, False, V3AbIneligibleReason.unsupported_block),+ (_totp_blocks, False, V3AbIneligibleReason.block_totp_verification_url),+ (_no_reroutable_blocks, False, V3AbIneligibleReason.no_reroutable_blocks),+ (_eligible_mix_blocks, False, None),+ ],+ ids=[+ "script_run",+ "pinned_engine",+ "unsupported_block_type",+ "unsupported_block_validation_download",+ "block_totp_verification_url",+ "no_reroutable_blocks",+ "eligible",+ ],+)+def test_v3_ab_ineligibility_reason_maps_each_disqualifier(+ blocks_factory: Callable[[], list[BaseTaskBlock]],+ is_script_run: bool,+ expected_reason: V3AbIneligibleReason | None,+) -> None:+ blocks = blocks_factory()+ reason = v3_ab_ineligibility_reason(blocks, is_script_run=is_script_run)+ assert reason == expected_reason+ # run_is_eligible_for_v3_ab must stay a thin wrapper over the same decision.+ assert run_is_eligible_for_v3_ab(blocks, is_script_run=is_script_run) is (expected_reason is None)+++async def test_resolver_logs_the_ineligibility_reason(scoped_context: SkyvernContext) -> None:+ provider = _FakeExperimentationProvider({WORKFLOW_TASK_V3_AB_FLAG: True})+ with (+ patch(WORKFLOW_BLOCK_ENGINE_APP_TARGET) as mock_app,+ patch("skyvern.forge.sdk.experimentation.workflow_block_engine.LOG") as mock_log,+ ):+ mock_app.EXPERIMENTATION_PROVIDER = provider+ await resolve_workflow_block_engine_arm(+ scoped_context,+ workflow_run_id="wr_logged",+ organization_id="org_1",+ workflow_permanent_id="wpid_1",+ ineligibility_reason=V3AbIneligibleReason.pinned_engine,+ )++ assert scoped_context.workflow_block_engine_override is None+ resolved_call = mock_log.info.call_args+ assert resolved_call.args[0] == "Resolved workflow-block engine arm"+ assert resolved_call.kwargs["arm"] == "control"+ assert resolved_call.kwargs["ineligibility_reason"] == V3AbIneligibleReason.pinned_engine+ # The provider is never even queried for an ineligible run -- nothing to bucket.+ assert provider.calls == []