Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
Flip the Task V3 A/B coin before the code-mode decision; v3 runs never load or generate scripts (SKY-18016) (#8857)
pedrohsdbb0a79afcec6e
diff --git a/skyvern/core/script_generations/transform_workflow_run.py b/skyvern/core/script_generations/transform_workflow_run.pyindex eec2589f5..413933898 100644--- a/skyvern/core/script_generations/transform_workflow_run.py+++ b/skyvern/core/script_generations/transform_workflow_run.py@@ -6,6 +6,7 @@ import structlogfrom skyvern.core.script_generations.constants import SCRIPT_TASK_BLOCKSfrom skyvern.forge import app+from skyvern.schemas.run_enums import RunEnginefrom skyvern.schemas.workflows import BlockTypefrom skyvern.services import workflow_servicefrom skyvern.webeye.actions.action_types import ActionType@@ -14,6 +15,10 @@ from skyvern.webeye.actions.actions import Action, reasoning_is_turn_scopedLOG = structlog.get_logger(__name__)+class TaskV3RunNotScriptableError(Exception):+ """Task V3 actions carry no element data, so a script built from them has dead selectors."""++@dataclassclass CodeGenInput:file_name: str@@ -258,6 +263,10 @@ async def transform_workflow_run_to_code_gen_input(workflow_run_id: str, organizworkflow_run_id=workflow_run_id, organization_id=organization_id)workflow_run_blocks.sort(key=lambda x: x.created_at)+ # Checked on the same snapshot the actions are read through: a block row records its engine before any of+ # its actions exist, so an in-flight mint cannot read a v3 task's actions without also reading its v3 row.+ if any(block.engine == RunEngine.skyvern_v3 for block in workflow_run_blocks):+ raise TaskV3RunNotScriptableError(workflow_run_id)# Create mapping from definition blocks by label for quick lookupworkflow_run_blocks_by_label = {block.label: block for block in workflow_run_blocks if block.label}diff --git a/skyvern/forge/agent.py b/skyvern/forge/agent.pyindex 411b16fad..0cf8f184a 100644--- a/skyvern/forge/agent.py+++ b/skyvern/forge/agent.py@@ -3672,6 +3672,7 @@ class ForgeAgent:)return step, detailed_output, None+ dispatched_as_v3 = engine == RunEngine.skyvern_v3# A bare task always qualifies; a workflow block must be an allowed type.task_block_supports_v3 = task_block is None or _task_block_supports_v3(task_block)if engine == RunEngine.skyvern_v3 and task_block is not None and not task_block_supports_v3:@@ -3697,7 +3698,7 @@ class ForgeAgent:"DISABLE_TASK_V3 could not be evaluated; falling back to the step engine",task_id=task.task_id,workflow_run_id=task.workflow_run_id,- # The persisted run type and arm still read v3; cohort reads exclude these by this line.+ # The arm still reads v3; cohort reads exclude these by this line.route_reason="flag_error",exc_info=True,)@@ -3745,6 +3746,23 @@ class ForgeAgent:)return step, detailed_output, None+ if dispatched_as_v3 and task.workflow_run_id:+ # The block row was labeled v3 before dispatch, and script generation skips a run with a+ # v3 row, so a fallback left labeled v3 would lose a script the run was asked to generate.+ try:+ await app.DATABASE.observer.set_workflow_run_block_engine_by_task_id(+ task.task_id, RunEngine.skyvern_v1, organization_id=task.organization_id+ )+ except Exception:+ LOG.warning(+ "Could not record the step-engine fallback on the block row",+ task_id=task.task_id,+ workflow_run_id=task.workflow_run_id,+ exc_info=True,+ )+ # Later steps recurse with this engine; pinning v1 keeps them off the gate and this write.+ engine = RunEngine.skyvern_v1+if page := await browser_state.get_working_page():await self.register_async_operations(organization, task, page)diff --git a/skyvern/forge/sdk/db/repositories/observer.py b/skyvern/forge/sdk/db/repositories/observer.pyindex dd80e7f9d..b4dcbaaf4 100644--- a/skyvern/forge/sdk/db/repositories/observer.py+++ b/skyvern/forge/sdk/db/repositories/observer.py@@ -713,6 +713,41 @@ class ObserverRepository(BaseRepository):).first()return RunEngine(engine) if engine else None+ @db_operation("set_workflow_run_block_engine_by_task_id")+ async def set_workflow_run_block_engine_by_task_id(+ self,+ task_id: str,+ engine: RunEngine,+ organization_id: str | None = None,+ ) -> None:+ async with self.Session() as session:+ await session.execute(+ update(WorkflowRunBlockModel)+ .where(WorkflowRunBlockModel.task_id == task_id)+ .where(WorkflowRunBlockModel.organization_id == organization_id)+ .values(engine=engine.value)+ )+ await session.commit()++ @db_operation("workflow_run_has_block_on_engine")+ async def workflow_run_has_block_on_engine(+ self,+ workflow_run_id: str,+ engine: RunEngine,+ organization_id: str | None = None,+ ) -> bool:+ async with self.Session() as session:+ found = (+ await session.scalars(+ select(WorkflowRunBlockModel.workflow_run_block_id)+ .filter_by(workflow_run_id=workflow_run_id)+ .filter_by(organization_id=organization_id)+ .filter_by(engine=engine.value)+ .limit(1)+ )+ ).first()+ return found is not None+@db_operation("get_workflow_run_blocks")async def get_workflow_run_blocks(self,diff --git a/skyvern/forge/sdk/experimentation/workflow_block_engine.py b/skyvern/forge/sdk/experimentation/workflow_block_engine.pyindex 1629299b7..a0b1ee059 100644--- a/skyvern/forge/sdk/experimentation/workflow_block_engine.py+++ b/skyvern/forge/sdk/experimentation/workflow_block_engine.py@@ -23,6 +23,7 @@ if TYPE_CHECKING:LOG = structlog.get_logger()WORKFLOW_TASK_V3_AB_FLAG = "WORKFLOW_TASK_V3_AB"+WOULD_BE_CODE_PROPERTY = "would_be_code"DISABLE_TASK_V3_FLAG = "DISABLE_TASK_V3"# Trigger kinds whose run executes a workflow the platform minted for that one request rather than@@ -45,6 +46,10 @@ class WorkflowBlockEngineRouteReason(StrEnum):flag_bucket_treatment = "flag_bucket_treatment"flag_bucket_control = "flag_bucket_control"+ # The same buckets for a run that intended code mode: its control runs cached code or the agent the+ # code path falls back to, so it is a different comparison from the v3-vs-v1 cells above.+ code_bucket_treatment = "code_bucket_treatment"+ code_bucket_control = "code_bucket_control"new_self_serve_workflow_default = "new_self_serve_workflow_default"# A workflow born at or after TASK_V3_CHOSEN_ENGINE_CUTOFF, never randomized: at least one block# left its engine unset and runs on v3 (``new_workflow_v3_default``), or every block's engine was@@ -90,6 +95,9 @@ class WorkflowBlockEngineArmDecision:billing_tier: BillingTier | None = None# An explicit skyvern_v1 on a block is a pin, not a routable default (see Block.resolve_engine).honors_chosen_engine: bool = False+ # The run intended code mode when the arm was resolved; control keeps that path. None when no arm was+ # resolved, so a run whose attribution was lost never reads as an agent-mode run.+ would_be_code: bool | None = NoneNO_ARM_DECISION = WorkflowBlockEngineArmDecision()@@ -202,6 +210,7 @@ async def _ab_flag_puts_run_in_treatment(organization_id: str | None,workflow_permanent_id: str | None,billing_tier: BillingTier,+ would_be_code: bool,) -> bool | None:"""Whether the percentage knob buckets this run into treatment, or ``None`` if it never answered.@@ -221,6 +230,9 @@ async def _ab_flag_puts_run_in_treatment("organization_id": organization_id,"workflow_permanent_id": workflow_permanent_id or "not_workflow",BILLING_TIER_PROPERTY: billing_tier.value,+ # Sent for reads only. A condition on it would be inconclusive, so None (control), for any+ # evaluation that omits it: posthog's local match raises on a property it was not given.+ WOULD_BE_CODE_PROPERTY: "true" if would_be_code else "false",},)@@ -236,6 +248,7 @@ async def resolve_workflow_block_engine_arm(ineligibility_reason: V3AbIneligibleReason | None,takes_default_engine: bool | None,pinned_v1_blocks: int = 0,+ would_be_code: bool = False,) -> None:"""Resolve the workflow-block engine A/B once at execution start and pin the arm on the context.@@ -266,6 +279,12 @@ async def resolve_workflow_block_engine_arm(``pinned_v1_blocks`` counts blocks a person pinned to skyvern-1.0 (see ``pinned_v1_block_count``);eligibility already keeps such a run out of the A/B, and this labels it ``pinned_v1_engine``.++ ``would_be_code`` says the run intended code mode (a cached script, the no-script-yet code_generation+ mode, or a code-rollout upgrade), whether or not a script ends up executing. An explicit ``code_gen``+ request is not bucketed: it is ineligible. Only the percentage knob may move+ such a run: a treatment bucket runs it as a v3 agent without its script (``code_mode_displaced``),+ while every other route leaves it on the code path it had."""if context.workflow_block_engine_resolved_run_id == workflow_run_id:return@@ -331,7 +350,8 @@ async def resolve_workflow_block_engine_arm(# The tier, the status and the trigger are checked first, so anything that# cannot use the rule pays for no workflow read.takes_new_workflow_default = (- billing_tier == BillingTier.SELF_SERVE+ not would_be_code+ and billing_tier == BillingTier.SELF_SERVEand _is_kept_workflow_run(workflow_status, trigger_type)and await born_at_or_after(settings.TASK_V3_DEFAULT_ENGINE_WORKFLOW_CUTOFF))@@ -347,17 +367,26 @@ async def resolve_workflow_block_engine_arm(organization_id=organization_id,workflow_permanent_id=workflow_permanent_id,billing_tier=billing_tier,+ would_be_code=would_be_code,)if in_treatment:override = RunEngine.skyvern_v3- route_reason = WorkflowBlockEngineRouteReason.flag_bucket_treatment+ route_reason = (+ WorkflowBlockEngineRouteReason.code_bucket_treatment+ if would_be_code+ else WorkflowBlockEngineRouteReason.flag_bucket_treatment+ )elif in_treatment is None:# The flag never answered, so this run is on control without having been# randomized. Labelled apart from the bucket so the control cell every# per-arm read builds out of flag_bucket_control stays a randomized one.route_reason = WorkflowBlockEngineRouteReason.flag_undefinedelse:- route_reason = WorkflowBlockEngineRouteReason.flag_bucket_control+ route_reason = (+ WorkflowBlockEngineRouteReason.code_bucket_control+ if would_be_code+ else WorkflowBlockEngineRouteReason.flag_bucket_control+ )except Exception:LOG.warning("Failed to resolve the workflow-block engine arm; using control",@@ -371,6 +400,7 @@ async def resolve_workflow_block_engine_arm(route_reason=route_reason,billing_tier=billing_tier,honors_chosen_engine=honors_chosen_engine,+ would_be_code=would_be_code,)context.workflow_block_engine_override = overridecontext.workflow_block_engine_arm_decision = decision@@ -390,9 +420,22 @@ async def resolve_workflow_block_engine_arm(# doubles as "nobody looked".billing_tier=engine_arm_log_value(billing_tier),pinned_v1_blocks=pinned_v1_blocks,+ would_be_code=would_be_code,)+def code_mode_displaced(workflow_run_id: str | None) -> bool:+ """Whether the A/B bucketed this run, headed for a cached script, into treatment: it then runs every+ block as a v3 agent instead of the script."""+ if not workflow_run_id:+ return False+ context = skyvern_context.current()+ if context is None or context.workflow_block_engine_resolved_run_id != workflow_run_id:+ return False+ decision = context.workflow_block_engine_arm_decision+ return bool(decision and decision.route_reason == WorkflowBlockEngineRouteReason.code_bucket_treatment)++def workflow_block_engine_override(workflow_run_id: str | None) -> RunEngine | None:"""The engine the A/B pinned for this run, or None when the run is control or unresolved.diff --git a/skyvern/forge/sdk/workflow/models/block.py b/skyvern/forge/sdk/workflow/models/block.pyindex b37c62640..5b153271d 100644--- a/skyvern/forge/sdk/workflow/models/block.py+++ b/skyvern/forge/sdk/workflow/models/block.py@@ -2078,8 +2078,8 @@ class BaseTaskBlock(Block):def resolve_engine(self, workflow_run_id: str | None) -> RunEngine:"""The engine this block dispatches to, after the per-run A/B.- Both the persisted workflow_run_blocks.engine and the execute_step dispatch read this, so- the recorded engine cannot disagree with the one that ran. A block pinned to a non-default+ Both the persisted workflow_run_blocks.engine and the execute_step dispatch read this; when the+ dispatch falls back to the step engine, execute_step corrects the row to the engine that ran. A block pinned to a non-defaultengine is honored as-authored, and a block the eligibility check never saw is left alone;neither is ever rerouted. An unset engine routes like skyvern_v1, except in a run that honors thechosen engine or on a block whose skyvern_v1 a person pinned.@@ -2462,6 +2462,8 @@ class BaseTaskBlock(Block):workflow_run_block_id=workflow_run_block_id,task_id=task.task_id,organization_id=organization_id,+ # A retry reuses this row, which a step-engine fallback on the previous attempt relabeled.+ engine=self.resolve_engine(workflow_run_id).value,)current_running_task = taskorganization = await app.DATABASE.organizations.get_organization(@@ -8184,6 +8186,8 @@ async def wrapper({default_args}):attempt_number=workflow_run_context.attempt_number,label="Self-heal recovery",block_type=BlockType.TASK,+ # Dispatched on v3 below; script generation reads this column to keep v3 actions out.+ engine=RunEngine.skyvern_v3,)recovery_block_id = recovery_block.workflow_run_block_id@@ -19824,13 +19828,14 @@ class V3AbIneligibleReason(StrEnum):eligible" but not "why not.""""- script_run = "script_run"pinned_engine = "pinned_engine"+ # An explicit request to generate code from this run; v3 actions cannot produce a usable script.+ code_generation_requested = "code_generation_requested"unsupported_block = "unsupported_block"no_reroutable_blocks = "no_reroutable_blocks"-def v3_ab_ineligibility_reason(blocks: list[BlockTypeVar], *, is_script_run: bool) -> V3AbIneligibleReason | None:+def v3_ab_ineligibility_reason(blocks: list[BlockTypeVar]) -> V3AbIneligibleReason | None:"""Why a whole workflow run may not be rerouted onto v3 by the A/B, or None if it may.Eligibility is a property of the RUN, not of a block: a run whose blocks disagreed about the@@ -19847,12 +19852,9 @@ def v3_ab_ineligibility_reason(blocks: list[BlockTypeVar], *, is_script_run: boomixed-arm run this predicate exists to prevent -- and a partial re-run is not comparable to afull run in the cohort anyway.- Script runs are excluded: their blocks execute as cached code and never reach engine dispatch,- so treatment would land only on the ai_fallback subset -- the blocks that already failed cached- execution.+ A run headed for a cached script is eligible: treatment runs it as a v3 agent without the script+ (see ``code_mode_displaced``), so the arm covers every block rather than only the ai_fallback ones."""- if is_script_run:- return V3AbIneligibleReason.script_runreroutable_blocks = 0for block in blocks:if isinstance(block, ConditionalBlock):@@ -19919,9 +19921,9 @@ def takes_default_engine(blocks: list[BlockTypeVar]) -> bool | None:return False if has_engine_block else None-def run_is_eligible_for_v3_ab(blocks: list[BlockTypeVar], *, is_script_run: bool) -> bool:+def run_is_eligible_for_v3_ab(blocks: list[BlockTypeVar]) -> bool:"""Whether a whole workflow run may be rerouted onto v3 by the A/B; see v3_ab_ineligibility_reason."""- return v3_ab_ineligibility_reason(blocks, is_script_run=is_script_run) is None+ return v3_ab_ineligibility_reason(blocks) is Nonedef get_all_blocks(blocks: list[BlockTypeVar]) -> list[BlockTypeVar]:diff --git a/skyvern/forge/sdk/workflow/service.py b/skyvern/forge/sdk/workflow/service.pyindex 7f62bfa26..540ac8775 100644--- a/skyvern/forge/sdk/workflow/service.py+++ b/skyvern/forge/sdk/workflow/service.py@@ -132,6 +132,7 @@ from skyvern.forge.sdk.experimentation.transient_ui_capture import resolve_transfrom skyvern.forge.sdk.experimentation.workflow_block_engine import (ARM_ATTRIBUTION_LOST,WorkflowBlockEngineArmAttribution,+ code_mode_displaced,engine_arm_log_value,resolve_workflow_block_engine_arm,resolved_workflow_block_engine_arm_attribution,@@ -7706,6 +7707,7 @@ class WorkflowService:if (is_adaptive_caching(workflow, workflow_run)and is_script_execution+ and not await self._ran_a_block_on_v3(workflow_run)and pre_finally_statusnot in (WorkflowRunStatus.canceled,@@ -8158,7 +8160,11 @@ class WorkflowService:# share an arm. Covers the DAG executor too, which this method delegates to.current_context = skyvern_context.current()if current_context:- ineligibility_reason = v3_ab_ineligibility_reason(all_blocks, is_script_run=is_script_run)+ ineligibility_reason = (+ V3AbIneligibleReason.code_generation_requested+ if workflow_run.code_gen+ else v3_ab_ineligibility_reason(all_blocks)+ )await resolve_workflow_block_engine_arm(current_context,workflow_run_id=workflow_run_id,@@ -8167,13 +8173,16 @@ class WorkflowService:workflow_status=workflow.status,trigger_type=workflow_run.trigger_type,ineligibility_reason=ineligibility_reason,- takes_default_engine=takes_default_engine(all_blocks),+ # A code_gen request has no engine to choose: the chosen-engine rule would put it on v3,+ # whose actions cannot yield the script it asked for.+ takes_default_engine=None if workflow_run.code_gen else takes_default_engine(all_blocks),# Counted only when the pin is what left the A/B, so pinned_v1_engine never labels another reason.pinned_v1_blocks=(pinned_v1_block_count(all_blocks)if ineligibility_reason == V3AbIneligibleReason.pinned_engineelse 0),+ would_be_code=is_script_run,)else:LOG.warning(@@ -8181,6 +8190,18 @@ class WorkflowService:workflow_run_id=workflow_run_id,)+ if is_script_run and code_mode_displaced(workflow_run_id):+ LOG.info(+ "A/B treatment runs this code-mode run on v3 as an agent, without its script",+ workflow_run_id=workflow_run_id,+ script_id=script.script_id if script else None,+ )+ script = None+ script_is_pinned = False+ is_script_run = False+ if current_context:+ current_context.generate_script = False+# A cached script can be superseded (e.g. a stale pin replaced by a platform static# script) — drop it here so the ensure_static_script path below re-creates the pin.if script and is_script_run:@@ -8665,6 +8686,24 @@ class WorkflowService:name=f"script_gen_{workflow_run.workflow_run_id}",)+ async def _ran_a_block_on_v3(self, workflow_run: WorkflowRun) -> bool:+ """Task V3 records no element data on its actions, so a script generated from a run that used it+ carries dead selectors, and later code-mode runs of the workflow would load it."""+ try:+ return await app.DATABASE.observer.workflow_run_has_block_on_engine(+ workflow_run_id=workflow_run.workflow_run_id,+ engine=RunEngine.skyvern_v3,+ organization_id=workflow_run.organization_id,+ )+ except Exception:+ # Unknown counts as v3: a skipped script is regenerated by the next v1 run, a dead one is loaded.+ LOG.warning(+ "Could not read the run's block engines; skipping script work",+ workflow_run_id=workflow_run.workflow_run_id,+ exc_info=True,+ )+ return True+async def _do_generate_pending_script(self,workflow: Workflow,@@ -8672,6 +8711,8 @@ class WorkflowService:) -> None:"""Fire-and-forget wrapper for pending script generation with error handling."""try:+ if await self._ran_a_block_on_v3(workflow_run):+ returnawait workflow_script_service.generate_or_update_pending_workflow_script(workflow_run=workflow_run,workflow=workflow,@@ -13437,6 +13478,7 @@ class WorkflowService:# run that was never randomized. Per-arm reads filter on route_reason, and the tier# is what the non-enterprise ramp steps on.route_reason=engine_arm_log_value(arm_decision.route_reason),+ would_be_code=arm_decision.would_be_code,billing_tier=engine_arm_log_value(arm_decision.billing_tier),)# Run minutes measure compute. A run finalized without ever reaching@@ -17448,6 +17490,14 @@ class WorkflowService:# request is not madereturn+ if await self._ran_a_block_on_v3(workflow_run):+ LOG.info(+ "Skipping script generation: this run executed a block on Task V3",+ workflow_run_id=workflow_run.workflow_run_id,+ workflow_permanent_id=workflow.workflow_permanent_id,+ )+ return+existing_script, rendered_cache_key_value, _is_pinned = await workflow_script_service.get_workflow_script(workflow,workflow_run,diff --git a/skyvern/services/script_service.py b/skyvern/services/script_service.pyindex e4af6973f..10aa04521 100644--- a/skyvern/services/script_service.py+++ b/skyvern/services/script_service.py@@ -1745,6 +1745,15 @@ async def _fallback_to_ai_run(complete_verification=complete_verification,include_action_history_in_verification=include_action_history_in_verification,)+ if workflow_run_block_id and engine == RunEngine.skyvern_v3:+ # The row was created for cached code with no engine; script generation and the reviewer+ # read this column to keep a v3 run's actions out of the workflow's script. Not swallowed:+ # a v3 fallback whose row still reads v1 could be minted into a dead-selector script.+ await app.DATABASE.observer.update_workflow_run_block(+ workflow_run_block_id=workflow_run_block_id,+ organization_id=organization_id,+ engine=engine.value,+ )await app.agent.execute_step(organization=organization,task=task,diff --git a/skyvern/services/workflow_script_service.py b/skyvern/services/workflow_script_service.pyindex 8a0edee94..f62366c1f 100644--- a/skyvern/services/workflow_script_service.py+++ b/skyvern/services/workflow_script_service.py@@ -18,7 +18,10 @@ from skyvern.core.script_generations.generate_script import (ScriptBlockSource,generate_workflow_script_python_code,)-from skyvern.core.script_generations.transform_workflow_run import transform_workflow_run_to_code_gen_input+from skyvern.core.script_generations.transform_workflow_run import (+ TaskV3RunNotScriptableError,+ transform_workflow_run_to_code_gen_input,+)from skyvern.forge import appfrom skyvern.forge.sdk.cache.factory import CacheFactoryfrom skyvern.forge.sdk.core import skyvern_context@@ -1053,6 +1056,13 @@ async def generate_workflow_script(use_semantic_selectors=adaptive,adaptive_caching=adaptive,)+ except TaskV3RunNotScriptableError:+ LOG.info(+ "Skipping script generation: this run executed a block on Task V3",+ workflow_run_id=workflow_run.workflow_run_id,+ workflow_permanent_id=workflow.workflow_permanent_id,+ )+ returnexcept Exception:generation_duration_ms = (time.monotonic() - generation_start) * 1000LOG.error(diff --git a/tests/unit/_workflow_block_engine_fakes.py b/tests/unit/_workflow_block_engine_fakes.pyindex 1e3b33588..28e331b56 100644--- a/tests/unit/_workflow_block_engine_fakes.py+++ b/tests/unit/_workflow_block_engine_fakes.py@@ -119,6 +119,7 @@ async def resolve_arm(workflow_status: WorkflowStatus = WorkflowStatus.published,trigger_type: WorkflowRunTriggerType | None = WorkflowRunTriggerType.api,takes_default_engine: bool | None = True,+ would_be_code: bool = False,) -> Resolution:async def read_birth_timestamp(workflow_permanent_id: str, organization_id: str) -> datetime | None:if first_version_error is not None:@@ -163,6 +164,7 @@ async def resolve_arm(trigger_type=trigger_type,ineligibility_reason=ineligibility_reason,takes_default_engine=takes_default_engine,+ would_be_code=would_be_code,)logged = dict(mock_log.info.call_args.kwargs) if mock_log.info.call_args else {}return Resolution(log=logged, birth_reads=birth_reads, warnings=list(mock_log.warning.call_args_list))diff --git a/tests/unit/test_agent_task_v3.py b/tests/unit/test_agent_task_v3.pyindex 8c02b9814..4c0940f24 100644--- a/tests/unit/test_agent_task_v3.py+++ b/tests/unit/test_agent_task_v3.py@@ -1930,6 +1930,7 @@ async def _run_execute_step_gate(workflow_run: Any = None,workflow_owned_recovery: bool = False,recovery_code_progress: CodeProgressRecord | None = None,+ observer: Any = None,**task_overrides: Any,) -> tuple[AsyncMock, AsyncMock]:"""Drive ForgeAgent.execute_step through the v3 dispatch gate and return (mocked _execute_task_v3,@@ -1967,6 +1968,10 @@ async def _run_execute_step_gate(mock_app.DATABASE.tasks.get_task = AsyncMock(return_value=None)mock_app.DATABASE.tasks.update_task = AsyncMock()mock_app.DATABASE.workflow_runs.get_workflow_run = AsyncMock(return_value=workflow_run)+ if observer is not None:+ mock_app.DATABASE.observer = observer+ else:+ mock_app.DATABASE.observer.set_workflow_run_block_engine_by_task_id = AsyncMock()mock_app.AGENT_FUNCTION.validate_step_execution = AsyncMock()if experimentation_provider is not None:mock_app.EXPERIMENTATION_PROVIDER = experimentation_provider@@ -2124,7 +2129,7 @@ async def test_an_unevaluable_kill_switch_runs_an_explicit_v3_block_on_the_step_# The engine handed down is the one every later step of this task recurses with, so a v3 pin# here would re-read the flag next step and could restart the task on v3 mid-way.assert step_engine_mock.await_args.kwargs["engine"] == agent_module.RunEngine.skyvern_v1- # The run's persisted type and arm still read v3, so this line is what cohort reads exclude it by.+ # The run's arm still reads v3, so this line is what cohort reads exclude it by.assert any(log.get("route_reason") == "flag_error" and log.get("workflow_run_id") == "wr_task_v3_kill_down" for log in logs)diff --git a/tests/unit/test_missing_starter_url.py b/tests/unit/test_missing_starter_url.pyindex 844684480..dff345280 100644--- a/tests/unit/test_missing_starter_url.py+++ b/tests/unit/test_missing_starter_url.py@@ -128,6 +128,7 @@ def _mock_block_execute_deps(yield {"task": task,"tasks_db": tasks_db,+ "observer_db": observer_db,"agent": agent,"browser_state": browser_state,"browser_manager": browser_manager,diff --git a/tests/unit/test_pending_script_promotion.py b/tests/unit/test_pending_script_promotion.pyindex 2d456ca8b..26371ec96 100644--- a/tests/unit/test_pending_script_promotion.py+++ b/tests/unit/test_pending_script_promotion.py@@ -23,12 +23,17 @@ from skyvern.core.script_generations.generate_script import _build_block_statemefrom skyvern.forge import appfrom skyvern.forge.sdk.core import skyvern_contextfrom skyvern.forge.sdk.core.skyvern_context import SkyvernContext-from skyvern.forge.sdk.workflow.models.block import ExtractionBlock+from skyvern.forge.sdk.experimentation.providers import BaseExperimentationProvider+from skyvern.forge.sdk.workflow.models.block import ExtractionBlock, TaskBlockfrom skyvern.forge.sdk.workflow.models.parameter import OutputParameterfrom skyvern.forge.sdk.workflow.service import WorkflowService+from skyvern.schemas.run_enums import RunEnginefrom skyvern.schemas.scripts import ScriptStatusfrom skyvern.schemas.workflows import BlockStatus, BlockTypefrom skyvern.services import workflow_script_service+from skyvern.services import workflow_service as workflow_service_module+from skyvern.webeye.actions.actions import ClickAction+from tests.unit.test_agent_task_v3 import _make_block, _run_execute_step_gatedef make_workflow(block_types: list[BlockType]) -> SimpleNamespace:@@ -378,9 +383,40 @@ class TestPendingMintUsesOriginalRevision:scripts.create_workflow_script.assert_not_awaited()+def run_blocks_db(*engines: RunEngine | None) -> SimpleNamespace:+ async def has_block_on_engine(*, workflow_run_id: str, engine: RunEngine, organization_id: str) -> bool:+ return engine in engines++ return SimpleNamespace(workflow_run_has_block_on_engine=has_block_on_engine)+++class BlockRowsByTask:+ """Block-row engines keyed by task, written by the dispatch fallback and read by the mint guard."""++ def __init__(self, **engines: RunEngine) -> None:+ self.engines = dict(engines)++ async def set_workflow_run_block_engine_by_task_id(+ self, task_id: str, engine: RunEngine, organization_id: str | None = None+ ) -> None:+ self.engines[task_id] = engine++ async def workflow_run_has_block_on_engine(+ self, *, workflow_run_id: str, engine: RunEngine, organization_id: str | None+ ) -> bool:+ return engine in self.engines.values()++class TestFinalizeReusesPendingScript:- def _patch_common(self, monkeypatch: pytest.MonkeyPatch, scripts: MagicMock) -> AsyncMock:- monkeypatch.setattr(app, "DATABASE", SimpleNamespace(scripts=scripts), raising=False)+ def _patch_common(+ self,+ monkeypatch: pytest.MonkeyPatch,+ scripts: MagicMock,+ *engines: RunEngine | None,+ observer: BlockRowsByTask | None = None,+ ) -> AsyncMock:+ database = SimpleNamespace(scripts=scripts, observer=observer or run_blocks_db(*engines))+ monkeypatch.setattr(app, "DATABASE", database, raising=False)monkeypatch.setattr(app, "ARTIFACT_MANAGER", SimpleNamespace(upload_aiotasks_map={}), raising=False)monkeypatch.setattr(workflow_script_service,@@ -457,3 +493,168 @@ class TestFinalizeReusesPendingScript:scripts.create_script.assert_awaited_once()assert generate_mock.await_args.kwargs["script"].script_id == "s_new"+++class TestNoScriptFromATaskV3Run:+ """Task V3 actions carry no element data, so a script minted from them has dead selectors, and the+ workflow's later code-mode runs would load it. Both mint paths must refuse such a run."""++ @pytest.mark.parametrize(+ ("engines", "mints"),+ [+ ((RunEngine.skyvern_v1, RunEngine.skyvern_v1), True),+ ((RunEngine.skyvern_v1, RunEngine.skyvern_v3), False),+ ((RunEngine.skyvern_v3,), False),+ ],+ )+ @pytest.mark.asyncio+ async def test_finalize_mints_only_from_a_run_with_no_v3_block(+ self, monkeypatch: pytest.MonkeyPatch, engines: tuple[RunEngine, ...], mints: bool+ ) -> None:+ scripts = make_scripts_db(pending_row=None)+ generate_mock = TestFinalizeReusesPendingScript()._patch_common(monkeypatch, scripts, *engines)++ await WorkflowService().generate_script_if_needed(+ workflow=make_workflow([BlockType.GOTO_URL, BlockType.TASK]),+ workflow_run=make_workflow_run(),+ finalize=True,+ )++ assert scripts.create_script.await_count == int(mints)+ assert generate_mock.await_count == int(mints)++ @pytest.mark.parametrize(("engine", "mints"), [(RunEngine.skyvern_v1, True), (RunEngine.skyvern_v3, False)])+ @pytest.mark.asyncio+ async def test_pending_mint_skips_a_run_with_a_v3_block(+ self, monkeypatch: pytest.MonkeyPatch, engine: RunEngine, mints: bool+ ) -> None:+ monkeypatch.setattr(app, "DATABASE", SimpleNamespace(observer=run_blocks_db(engine)), raising=False)+ pending_mint = AsyncMock()+ monkeypatch.setattr(workflow_script_service, "generate_or_update_pending_workflow_script", pending_mint)++ await WorkflowService()._do_generate_pending_script(make_workflow([BlockType.TASK]), make_workflow_run())++ assert pending_mint.await_count == int(mints)++ @pytest.mark.parametrize(("kill_switch_on", "mints"), [(True, True), (False, False)])+ @pytest.mark.asyncio+ async def test_a_v3_pinned_block_the_kill_switch_ran_on_v1_still_gets_its_requested_script(+ self, monkeypatch: pytest.MonkeyPatch, kill_switch_on: bool, mints: bool+ ) -> None:+ workflow_run = make_workflow_run()+ workflow_run.code_gen = True+ # execute_safe labels the row with the resolved engine before dispatch.+ rows = BlockRowsByTask(**{"task-123": RunEngine.skyvern_v3})+ provider = MagicMock(spec=BaseExperimentationProvider)+ provider.resolve_feature_flag_strict = AsyncMock(return_value=kill_switch_on)++ await _run_execute_step_gate(+ engine=RunEngine.skyvern_v3,+ task_block=_make_block(TaskBlock, label="pinned", engine=RunEngine.skyvern_v3),+ experimentation_provider=provider,+ observer=rows,+ workflow_run_id=workflow_run.workflow_run_id,+ )+ scripts = make_scripts_db(pending_row=None)+ generate_mock = TestFinalizeReusesPendingScript()._patch_common(monkeypatch, scripts, observer=rows)++ await WorkflowService().generate_script_if_needed(+ workflow=make_workflow([BlockType.GOTO_URL, BlockType.TASK]),+ workflow_run=workflow_run,+ )++ assert generate_mock.await_count == int(mints)++ @pytest.mark.asyncio+ async def test_a_non_v3_dispatch_keeps_its_row_and_engine(self) -> None:+ rows = BlockRowsByTask(**{"task-123": RunEngine.openai_cua})++ _v3_mock, step_engine_mock = await _run_execute_step_gate(+ engine=RunEngine.openai_cua,+ task_block=_make_block(TaskBlock, label="cua", engine=RunEngine.openai_cua),+ observer=rows,+ workflow_run_id=make_workflow_run().workflow_run_id,+ )++ assert rows.engines == {"task-123": RunEngine.openai_cua}+ assert step_engine_mock.await_args.kwargs["engine"] == RunEngine.openai_cua++ @pytest.mark.parametrize(+ ("second_block_engine", "mints"), [(RunEngine.skyvern_v1, True), (RunEngine.skyvern_v3, False)]+ )+ @pytest.mark.asyncio+ async def test_an_in_flight_pending_mint_whose_snapshot_holds_v3_actions_writes_nothing(+ self, monkeypatch: pytest.MonkeyPatch, second_block_engine: RunEngine, mints: bool+ ) -> None:+ # The mint passed its v3 check before the second block's row existed; by the snapshot that block's+ # actions are stored, so only the snapshot itself can refuse them.+ def run_block(label: str, task_id: str, engine: RunEngine, created_at: int) -> SimpleNamespace:+ return SimpleNamespace(+ workflow_run_block_id=f"wrb_{label}",+ parent_workflow_run_block_id=None,+ block_type=BlockType.TASK,+ label=label,+ task_id=task_id,+ status="completed",+ output={},+ created_at=created_at,+ engine=engine,+ )++ definition_blocks = [+ SimpleNamespace(label=label, block_type=BlockType.TASK, model_dump=lambda label=label: {"label": label})+ for label in ("cached", "pinned")+ ]+ workflow = SimpleNamespace(+ workflow_id="w_1",+ workflow_permanent_id="wpid_1",+ organization_id="o_1",+ title="wf",+ workflow_definition=SimpleNamespace(blocks=definition_blocks),+ model_dump=lambda: {"workflow_id": "w_1"},+ )+ run_request = SimpleNamespace(workflow_id="wpid_1", model_dump=lambda: {"workflow_id": "wpid_1"})+ tasks = [+ SimpleNamespace(task_id=t, model_dump=lambda t=t: {"task_id": t}) for t in ("tsk_cached", "tsk_pinned")+ ]+ actions = [+ ClickAction(action_id=f"a_{t}", task_id=t, element_id=f"el_{t}") for t in ("tsk_pinned", "tsk_cached")+ ]+ database = SimpleNamespace(+ observer=SimpleNamespace(+ get_workflow_run_blocks=AsyncMock(+ return_value=[+ run_block("cached", "tsk_cached", RunEngine.skyvern_v1, 1),+ run_block("pinned", "tsk_pinned", second_block_engine, 2),+ ]+ )+ ),+ tasks=SimpleNamespace(+ get_tasks_by_ids=AsyncMock(return_value=tasks), get_tasks_actions=AsyncMock(return_value=actions)+ ),+ )+ monkeypatch.setattr(app, "DATABASE", database, raising=False)+ monkeypatch.setattr(+ app,+ "WORKFLOW_SERVICE",+ SimpleNamespace(get_workflow_by_permanent_id=AsyncMock(return_value=workflow)),+ raising=False,+ )+ monkeypatch.setattr(+ workflow_service_module,+ "get_workflow_run_response",+ AsyncMock(return_value=SimpleNamespace(run_request=run_request)),+ )+ monkeypatch.setattr(workflow_script_service, "is_adaptive_caching", lambda *_: False)+ codegen = AsyncMock(side_effect=RuntimeError("stop after the generation input is accepted"))+ monkeypatch.setattr(workflow_script_service, "generate_workflow_script_python_code", codegen)++ await workflow_script_service.generate_workflow_script(+ workflow_run=make_workflow_run(),+ workflow=workflow,+ script=SimpleNamespace(script_id="s_1", script_revision_id="sr_1", version=1),+ rendered_cache_key_value="default:site",+ pending=True,+ )++ assert codegen.await_count == int(mints)diff --git a/tests/unit/test_script_service_fallback_engine.py b/tests/unit/test_script_service_fallback_engine.pyindex b7bbdf4d8..1bc0be89a 100644--- a/tests/unit/test_script_service_fallback_engine.py+++ b/tests/unit/test_script_service_fallback_engine.py@@ -266,7 +266,7 @@ async def test_fallback_of_an_unset_engine_block_follows_the_chosen_engine_cutofcontext,FakeExperimentationProvider(),workflow_run_id="wr_test",- ineligibility_reason=v3_ab_ineligibility_reason(blocks, is_script_run=True),+ ineligibility_reason=v3_ab_ineligibility_reason(blocks),takes_default_engine=takes_default_engine(blocks),first_version_created_at=born_at,)@@ -333,7 +333,7 @@ async def test_an_uncached_script_block_runs_where_its_stored_block_would_only_pcontext,FakeExperimentationProvider(),workflow_run_id="wr_test",- ineligibility_reason=v3_ab_ineligibility_reason(workflow.workflow_definition.blocks, is_script_run=True),+ ineligibility_reason=v3_ab_ineligibility_reason(workflow.workflow_definition.blocks),takes_default_engine=takes_default_engine(workflow.workflow_definition.blocks),first_version_created_at=born_at,)@@ -546,3 +546,62 @@ async def test_fallback_episode_excludes_decision_row_from_agent_action_count()update_episode.await_args.kwargs["agent_actions"]["failure_reason"]== script_service.VERIFIER_SWAP_FAILURE_REASON)+++class _DispatchReached(Exception):+ pass+++ ("block_engine", "recorded"), [(RunEngine.skyvern_v3, ["skyvern-3.0"]), (RunEngine.skyvern_v1, [])]+)+async def test_a_v3_fallback_records_its_engine_on_the_cached_blocks_row(+ block_engine: RunEngine, recorded: list[str]+) -> None:+ # The row was created for cached code with no engine. Script generation and the reviewer read this+ # column to keep a v3 run's selector-less actions out of the workflow's script.+ workflow = _make_workflow([_make_task_block("my_block", engine=block_engine)])+ app = _make_app(workflow)+ app.DATABASE.observer.update_workflow_run_block = AsyncMock()+ # The engine is written before dispatch; stop there instead of faking everything after it.+ app.agent.execute_step = AsyncMock(side_effect=_DispatchReached())+ with (+ patch(f"{MODULE}.app", app),+ patch(f"{MODULE}.skyvern_context.current", return_value=_make_context()),+ pytest.raises(_DispatchReached),+ ):+ await script_service._fallback_to_ai_run(+ block_type=BlockType.NAVIGATION,+ cache_key="my_block",+ prompt="do the thing",+ workflow_run_block_id="wrb_cached",+ )++ engines = [+ call.kwargs["engine"]+ for call in app.DATABASE.observer.update_workflow_run_block.call_args_list+ if "engine" in call.kwargs+ ]+ assert engines == recorded+++async def test_a_v3_fallback_that_cannot_record_its_engine_does_not_run() -> None:+ # Running anyway would leave a v3 run whose rows read v1, which script generation would then mint.+ workflow = _make_workflow([_make_task_block("my_block", engine=RunEngine.skyvern_v3)])+ app = _make_app(workflow)+ app.DATABASE.observer.update_workflow_run_block = AsyncMock(side_effect=RuntimeError("db unavailable"))+ with (+ patch(f"{MODULE}.app", app),+ patch(f"{MODULE}.skyvern_context.current", return_value=_make_context()),+ pytest.raises(RuntimeError),+ ):+ await script_service._fallback_to_ai_run(+ block_type=BlockType.NAVIGATION,+ cache_key="my_block",+ prompt="do the thing",+ workflow_run_block_id="wrb_cached",+ )++ app.agent.execute_step.assert_not_awaited()diff --git a/tests/unit/test_workflow_block_engine.py b/tests/unit/test_workflow_block_engine.pyindex b6abfc4c6..adfe4fd69 100644--- a/tests/unit/test_workflow_block_engine.py+++ b/tests/unit/test_workflow_block_engine.py@@ -25,12 +25,15 @@ from skyvern.forge.sdk.experimentation.providers import BaseExperimentationProvifrom skyvern.forge.sdk.experimentation.workflow_block_engine import (DISABLE_TASK_V3_FLAG,WORKFLOW_TASK_V3_AB_FLAG,+ WOULD_BE_CODE_PROPERTY,WorkflowBlockEngineRouteReason,_as_utc,+ code_mode_displaced,effective_default_engine,resolve_workflow_block_engine_arm,workflow_block_engine_override,)+from skyvern.forge.sdk.schemas.tasks import TaskStatusfrom skyvern.forge.sdk.workflow.models.block import (ActionBlock,BaseTaskBlock,@@ -121,7 +124,7 @@ def test_mixed_eligibility_run_download_block_is_eligible(scoped_context: Skyverdownload_block = _make_block(ActionBlock, label="dl", complete_on_download=True)blocks: list[BaseTaskBlock] = [eligible_1, eligible_2, download_block]- assert run_is_eligible_for_v3_ab(blocks, is_script_run=False) is True+ assert run_is_eligible_for_v3_ab(blocks) is Truedef test_mixed_eligibility_run_file_download_block_is_eligible(scoped_context: SkyvernContext) -> None:@@ -130,7 +133,7 @@ def test_mixed_eligibility_run_file_download_block_is_eligible(scoped_context: Sfile_download_block = _make_block(FileDownloadBlock, label="fd")blocks: list[BaseTaskBlock] = [eligible_1, eligible_2, file_download_block]- assert run_is_eligible_for_v3_ab(blocks, is_script_run=False) is True+ assert run_is_eligible_for_v3_ab(blocks) is Truedef test_mixed_eligibility_run_download_gated_validation_block_is_not_eligible(@@ -143,7 +146,7 @@ def test_mixed_eligibility_run_download_gated_validation_block_is_not_eligible(download_gated_validation = _make_block(ValidationBlock, label="dlv", complete_on_download=True)blocks: list[BaseTaskBlock] = [eligible_1, eligible_2, download_gated_validation]- assert run_is_eligible_for_v3_ab(blocks, is_script_run=False) is False+ assert run_is_eligible_for_v3_ab(blocks) is False@pytest.mark.asyncio@@ -154,7 +157,7 @@ async def test_mixed_eligibility_run_pins_whole_run_to_control(scoped_context: Sineligible_block.block_type = BlockType.WAITblocks: list[BaseTaskBlock] = [eligible_1, eligible_2, ineligible_block]- assert run_is_eligible_for_v3_ab(blocks, is_script_run=False) is False+ assert run_is_eligible_for_v3_ab(blocks) is Falseprovider = FakeExperimentationProvider({WORKFLOW_TASK_V3_AB_FLAG: True})await resolve_arm(@@ -191,7 +194,7 @@ def test_pinned_non_default_engine_block_disqualifies_the_run(pinned_engine: Run# in explicitly, not a treatment exposure.pinned = _make_block(NavigationBlock, label="pinned", navigation_goal="Apply to the job", engine=pinned_engine)- assert run_is_eligible_for_v3_ab([eligible, pinned], is_script_run=False) is False+ assert run_is_eligible_for_v3_ab([eligible, pinned]) is False@pytest.mark.asyncio@@ -201,7 +204,7 @@ async def test_all_eligible_run_resolves_every_block_to_treatment(scoped_context_make_block(NavigationBlock, label="t2", navigation_goal="Apply to the job"),_make_block(ActionBlock, label="t3"),]- assert run_is_eligible_for_v3_ab(blocks, is_script_run=False) is True+ assert run_is_eligible_for_v3_ab(blocks) is Trueprovider = FakeExperimentationProvider({WORKFLOW_TASK_V3_AB_FLAG: True})await resolve_arm(scoped_context, provider, workflow_run_id="wr_treatment", ineligibility_reason=None)@@ -364,14 +367,6 @@ async def test_an_ab_flag_local_evaluation_cannot_answer_is_control_but_not_a_buassert resolution.log["route_reason"] == WorkflowBlockEngineRouteReason.flag_undefined-def test_script_run_is_never_eligible() -> None:- blocks: list[BaseTaskBlock] = [- _make_block(TaskBlock, label="t1"),- _make_block(NavigationBlock, label="t2", navigation_goal="Apply to the job"),- ]- assert run_is_eligible_for_v3_ab(blocks, is_script_run=True) is False--@pytest.mark.asyncioasync def test_noop_provider_never_queried_and_leaves_engine_unchanged(scoped_context: SkyvernContext) -> None:provider = NoOpExperimentationProvider()@@ -401,7 +396,7 @@ def test_non_task_blocks_ignored_but_nested_loop_task_blocks_considered() -> Noneligible = _make_block(TaskBlock, label="ok")flat_without_loop = get_all_blocks([code_block, eligible])- assert run_is_eligible_for_v3_ab(flat_without_loop, is_script_run=False) is True+ assert run_is_eligible_for_v3_ab(flat_without_loop) is Trueineligible_nested = _make_block(TaskBlock, label="nested_pinned", engine=RunEngine.openai_cua)loop = ForLoopBlock(@@ -412,7 +407,7 @@ def test_non_task_blocks_ignored_but_nested_loop_task_blocks_considered() -> Nonflat_with_loop = get_all_blocks([code_block, eligible, loop])assert ineligible_nested in flat_with_loop- assert run_is_eligible_for_v3_ab(flat_with_loop, is_script_run=False) is False+ assert run_is_eligible_for_v3_ab(flat_with_loop) is Falsedef test_inert_blocks_do_not_disqualify_an_otherwise_eligible_run() -> None:@@ -422,7 +417,7 @@ def test_inert_blocks_do_not_disqualify_an_otherwise_eligible_run() -> None:human_block = _make_block(HumanInteractionBlock, label="human")eligible = _make_block(TaskBlock, label="ok")- assert run_is_eligible_for_v3_ab([url_block, human_block, eligible], is_script_run=False) is True+ assert run_is_eligible_for_v3_ab([url_block, human_block, eligible]) is Truedef test_run_with_only_inert_or_non_task_blocks_is_not_eligible() -> None:@@ -430,7 +425,7 @@ def test_run_with_only_inert_or_non_task_blocks_is_not_eligible() -> None:url_block = _make_block(UrlBlock, label="goto", url="https://example.com")human_block = _make_block(HumanInteractionBlock, label="human")- assert run_is_eligible_for_v3_ab([code_block, url_block, human_block], is_script_run=False) is False+ assert run_is_eligible_for_v3_ab([code_block, url_block, human_block]) is False@pytest.mark.asyncio@@ -516,6 +511,63 @@ async def test_base_task_block_execute_dispatches_with_the_resolved_engine(scopeassert captured["engine"] == RunEngine.skyvern_v3+async def test_a_retry_after_a_step_engine_fallback_runs_v3_under_a_v3_row(scoped_context: SkyvernContext) -> None:+ # Two attempts share one row. Attempt 1 falls back to the step engine (an unevaluable kill switch),+ # which relabels the row v1 and fails; attempt 2 dispatches the pin on v3 again.+ block = _make_block(TaskBlock, label="pinned_retry", engine=RunEngine.skyvern_v3, max_retries=1)+ row = {"engine": RunEngine.skyvern_v3.value}+ dispatches: list[tuple[RunEngine, str]] = []++ async def _update_row(**kwargs: Any) -> Any:+ if kwargs.get("engine") is not None:+ row["engine"] = kwargs["engine"]+ return MagicMock()++ async def _execute_step(**kwargs: Any) -> Any:+ dispatches.append((kwargs["engine"], row["engine"]))+ if len(dispatches) == 1:+ row["engine"] = RunEngine.skyvern_v1.value+ return None+ raise _EngineCaptured()++ now = datetime.now(UTC)+ failed_task = make_task(now, make_organization(now), status=TaskStatus.failed, failure_reason="step failed")+ with _mock_block_execute_deps(working_page_url="https://example.com/dashboard") as deps:+ deps["agent"].execute_step = AsyncMock(side_effect=_execute_step)+ deps["observer_db"].update_workflow_run_block = AsyncMock(side_effect=_update_row)+ deps["tasks_db"].get_task = AsyncMock(return_value=failed_task)+ with (+ patch(+ "skyvern.forge.sdk.workflow.models.block.app.STORAGE",+ new=MagicMock(+ get_current_attempt_downloaded_files=AsyncMock(return_value=[]),+ get_downloaded_file_signature_aliases=MagicMock(return_value=()),+ ),+ create=True,+ ),+ patch(+ "skyvern.forge.sdk.workflow.models.block.app.WORKFLOW_SERVICE.get_recent_task_screenshot_artifacts",+ new=AsyncMock(return_value=[]),+ ),+ patch(+ "skyvern.forge.sdk.workflow.models.block.app.WORKFLOW_SERVICE.get_recent_workflow_screenshot_artifacts",+ new=AsyncMock(return_value=[]),+ ),+ pytest.raises(_EngineCaptured),+ ):+ await block.execute(+ workflow_run_id="wr_missing_starter_url_test",+ workflow_run_block_id="wrb_test",+ organization_id="o_test",+ )++ assert dispatches == [+ (RunEngine.skyvern_v3, RunEngine.skyvern_v3.value),+ (RunEngine.skyvern_v3, RunEngine.skyvern_v3.value),+ ]++async def _persisted_engine_from_execute_workflow_blocks(monkeypatch: pytest.MonkeyPatch,provider: BaseExperimentationProvider,@@ -525,6 +577,8 @@ async def _persisted_engine_from_execute_workflow_blocks(first_version_created_at: datetime | None = None,trigger_type: WorkflowRunTriggerType | None = WorkflowRunTriggerType.api,block: BaseTaskBlock | None = None,+ script: Any = None,+ code_gen: bool | None = None,) -> RunEngine:"""Drives the real WorkflowService._execute_workflow_blocks -> _execute_single_block ->Block.execute_safe chain for a single eligible TaskBlock and returns the engine it persisted,@@ -552,6 +606,7 @@ async def _persisted_engine_from_execute_workflow_blocks(workflow_run.run_with = Noneworkflow_run.retried_from_workflow_run_id = Noneworkflow_run.trigger_type = trigger_type+ workflow_run.code_gen = code_genorganization = MagicMock()organization.organization_id = "org_e2e"@@ -579,13 +634,14 @@ async def _persisted_engine_from_execute_workflow_blocks()service = WorkflowService()- monkeypatch.setattr(service, "should_run_script", AsyncMock(return_value=False))+ monkeypatch.setattr(service, "should_run_script", AsyncMock(return_value=script is not None))with pytest.raises(_EngineCaptured):await service._execute_workflow_blocks(workflow=workflow,workflow_run=workflow_run,organization=organization,+ script=script,)return captured["engine"]@@ -759,22 +815,20 @@ def _jinja_condition_while_loop_blocks() -> list[Block]:@pytest.mark.parametrize(- ("blocks_factory", "is_script_run", "expected_reason"),+ ("blocks_factory", "expected_reason"),[- (_eligible_mix_blocks, True, V3AbIneligibleReason.script_run),- (_pinned_engine_blocks, False, V3AbIneligibleReason.pinned_engine),- (_unsupported_block_type_blocks, False, V3AbIneligibleReason.unsupported_block),- (_download_gated_validation_blocks, False, V3AbIneligibleReason.unsupported_block),- (_totp_blocks, False, None),- (_no_reroutable_blocks, False, V3AbIneligibleReason.no_reroutable_blocks),- (_eligible_mix_blocks, False, None),- (_prompt_branch_conditional_only_blocks, False, None),- (_jinja_only_conditional_blocks, False, V3AbIneligibleReason.no_reroutable_blocks),- (_prompt_condition_while_loop_only_blocks, False, None),- (_jinja_condition_while_loop_blocks, False, V3AbIneligibleReason.no_reroutable_blocks),+ (_pinned_engine_blocks, V3AbIneligibleReason.pinned_engine),+ (_unsupported_block_type_blocks, V3AbIneligibleReason.unsupported_block),+ (_download_gated_validation_blocks, V3AbIneligibleReason.unsupported_block),+ (_totp_blocks, None),+ (_no_reroutable_blocks, V3AbIneligibleReason.no_reroutable_blocks),+ (_eligible_mix_blocks, None),+ (_prompt_branch_conditional_only_blocks, None),+ (_jinja_only_conditional_blocks, V3AbIneligibleReason.no_reroutable_blocks),+ (_prompt_condition_while_loop_only_blocks, None),+ (_jinja_condition_while_loop_blocks, V3AbIneligibleReason.no_reroutable_blocks),],ids=[- "script_run","pinned_engine","unsupported_block_type","unsupported_block_validation_download",@@ -789,14 +843,13 @@ def _jinja_condition_while_loop_blocks() -> list[Block]:)def test_v3_ab_ineligibility_reason_maps_each_disqualifier(blocks_factory: Callable[[], list[BaseTaskBlock]],- is_script_run: bool,expected_reason: V3AbIneligibleReason | None,) -> None:blocks = blocks_factory()- reason = v3_ab_ineligibility_reason(blocks, is_script_run=is_script_run)+ reason = v3_ab_ineligibility_reason(blocks)assert reason == expected_reason# run_is_eligible_for_v3_ab must stay a thin wrapper over the same decision.- assert run_is_eligible_for_v3_ab(blocks, is_script_run=is_script_run) is (expected_reason is None)+ assert run_is_eligible_for_v3_ab(blocks) is (expected_reason is None)@pytest.mark.asyncio@@ -1039,7 +1092,7 @@ async def test_a_run_that_never_reaches_the_ab_pays_for_no_tier_lookup(scoped_co# kill switch stops: a Redis or pooler incident is exactly when that switch gets flipped, and# this resolver holds a lock while it runs.provider = FakeExperimentationProvider({WORKFLOW_TASK_V3_AB_FLAG: True})- for reason, run_id in ((V3AbIneligibleReason.script_run, "wr_inelig"), (None, "wr_killed")):+ for reason, run_id in ((V3AbIneligibleReason.unsupported_block, "wr_inelig"), (None, "wr_killed")):provider.flags[DISABLE_TASK_V3_FLAG] = reason is Nonetier = AsyncMock(return_value=BillingTier.ENTERPRISE)with (@@ -1354,13 +1407,13 @@ async def test_ineligible_run_is_never_defaulted_to_v3(scoped_context: SkyvernContext, v3_default_cutoff: datetime) -> None:# Run-level eligibility is about whether the run can be rerouted at all, so the rule changes the- # arm decision only: a script run or a pinned block still executes as authored.+ # arm decision only: an unsupported or pinned block still executes as authored.provider = FakeExperimentationProvider({WORKFLOW_TASK_V3_AB_FLAG: True})resolution = await resolve_arm(scoped_context,provider,workflow_run_id="wr_new_but_ineligible",- ineligibility_reason=V3AbIneligibleReason.script_run,+ ineligibility_reason=V3AbIneligibleReason.unsupported_block,billing_tier=BillingTier.SELF_SERVE,first_version_created_at=v3_default_cutoff.replace(tzinfo=None) + timedelta(days=1),)@@ -1435,7 +1488,7 @@ async def _resolve_chosen_engine_run(context,provider,workflow_run_id=run_id,- ineligibility_reason=v3_ab_ineligibility_reason(blocks, is_script_run=False),+ ineligibility_reason=v3_ab_ineligibility_reason(blocks),takes_default_engine=takes_default_engine(blocks),billing_tier=BillingTier.ENTERPRISE,first_version_created_at=born_at,@@ -1577,7 +1630,7 @@ def test_an_omitted_engine_is_unset_and_a_stored_one_is_kept() -> None:assert block.engine is Noneassert type(block).model_validate(block.model_dump()).engine is None# Unset routes like skyvern_v1 before the cutoff, so it never reads as a pin.- assert v3_ab_ineligibility_reason([block], is_script_run=False) is None+ assert v3_ab_ineligibility_reason([block]) is Noneassert takes_default_engine([block]) is True@@ -1712,3 +1765,105 @@ def test_a_pinned_skyvern_v1_survives_the_stored_definition_and_counts_as_a_chanassert WorkflowDefinition.model_validate(pinned.model_dump(mode="json")).blocks[0].engine_pinned is True# Before the cutoff an unmarked skyvern-1.0 and an unset engine compare alike; a pin must not.assert workflow_definitions_differ(unmarked, pinned)+++ ("flags", "expected_reason", "displaced"),+ [+ ({WORKFLOW_TASK_V3_AB_FLAG: True}, WorkflowBlockEngineRouteReason.code_bucket_treatment, True),+ ({WORKFLOW_TASK_V3_AB_FLAG: False}, WorkflowBlockEngineRouteReason.code_bucket_control, False),+ ({}, WorkflowBlockEngineRouteReason.flag_undefined, False),+ ],+)+async def test_a_code_mode_run_takes_the_ab_coin_and_only_treatment_leaves_its_script(+ scoped_context: SkyvernContext,+ flags: dict[str, bool],+ expected_reason: WorkflowBlockEngineRouteReason,+ displaced: bool,+) -> None:+ provider = FakeExperimentationProvider(flags)++ resolution = await resolve_arm(+ scoped_context, provider, workflow_run_id="wr_code", ineligibility_reason=None, would_be_code=True+ )++ assert resolution.log["route_reason"] == expected_reason+ assert resolution.log["would_be_code"] is True+ assert scoped_context.workflow_block_engine_arm_decision.would_be_code is True+ assert code_mode_displaced("wr_code") is displaced+ assert code_mode_displaced("wr_other_run") is False+ # Sent for reads only; no release condition may target it (see _ab_flag_puts_run_in_treatment).+ [ab_properties] = [props for flag, _, props in provider.calls if flag == WORKFLOW_TASK_V3_AB_FLAG]+ assert ab_properties[WOULD_BE_CODE_PROPERTY] == "true"+++async def test_the_new_workflow_defaults_never_take_a_code_mode_run_off_its_script(+ scoped_context: SkyvernContext, monkeypatch: pytest.MonkeyPatch, v3_default_cutoff: datetime+) -> None:+ born_after = v3_default_cutoff.replace(tzinfo=None) + timedelta(days=1)+ self_serve = await resolve_arm(+ scoped_context,+ FakeExperimentationProvider({WORKFLOW_TASK_V3_AB_FLAG: False}),+ workflow_run_id="wr_self_serve_code",+ ineligibility_reason=None,+ billing_tier=BillingTier.SELF_SERVE,+ first_version_created_at=born_after,+ would_be_code=True,+ )+ assert self_serve.log["route_reason"] == WorkflowBlockEngineRouteReason.code_bucket_control++ monkeypatch.setattr(settings, "TASK_V3_CHOSEN_ENGINE_CUTOFF", v3_default_cutoff)+ chosen = await resolve_arm(+ scoped_context,+ FakeExperimentationProvider({WORKFLOW_TASK_V3_AB_FLAG: True}),+ workflow_run_id="wr_chosen_code",+ ineligibility_reason=None,+ first_version_created_at=born_after,+ would_be_code=True,+ )+ # The cutoff rule keeps routing a script run's uncached blocks and AI fallback, as before.+ assert chosen.log["route_reason"] == WorkflowBlockEngineRouteReason.new_workflow_v3_default+ assert code_mode_displaced("wr_self_serve_code") is False+ assert code_mode_displaced("wr_chosen_code") is False+++async def test_a_treatment_code_mode_run_executes_on_v3_without_loading_or_minting_a_script(+ scoped_context: SkyvernContext, monkeypatch: pytest.MonkeyPatch+) -> None:+ script = MagicMock(script_id="s_cached", script_revision_id="sr_cached")+ load_blocks = AsyncMock(return_value=[])+ monkeypatch.setattr(app.DATABASE.scripts, "get_script_blocks_by_script_revision_id", load_blocks)+ scoped_context.generate_script = True # what a code_gen run carries into execution++ engine = await _persisted_engine_from_execute_workflow_blocks(+ monkeypatch, FakeExperimentationProvider({WORKFLOW_TASK_V3_AB_FLAG: True}), script=script+ )++ assert engine == RunEngine.skyvern_v3+ load_blocks.assert_not_awaited()+ assert scoped_context.script_id is None+ assert scoped_context.generate_script is False++[email protected]("born_after_chosen_engine_cutoff", [False, True])+async def test_an_explicit_code_generation_request_stays_on_v1(+ scoped_context: SkyvernContext, monkeypatch: pytest.MonkeyPatch, born_after_chosen_engine_cutoff: bool+) -> None:+ # A run asked to produce a script must not land on v3, whose actions yield no usable one: neither by+ # the A/B nor by the chosen-engine rule for new workflows.+ born_at = None+ if born_after_chosen_engine_cutoff:+ monkeypatch.setattr(settings, "TASK_V3_CHOSEN_ENGINE_CUTOFF", CHOSEN_ENGINE_CUTOFF)+ born_at = CHOSEN_ENGINE_CUTOFF.replace(tzinfo=None) + timedelta(days=1)+ engine = await _persisted_engine_from_execute_workflow_blocks(+ monkeypatch,+ FakeExperimentationProvider({WORKFLOW_TASK_V3_AB_FLAG: True}),+ code_gen=True,+ first_version_created_at=born_at,+ )++ assert engine == RunEngine.skyvern_v1diff --git a/tests/unit/test_workflow_run_minutes_gate.py b/tests/unit/test_workflow_run_minutes_gate.pyindex c19db6491..1549aaa60 100644--- a/tests/unit/test_workflow_run_minutes_gate.py+++ b/tests/unit/test_workflow_run_minutes_gate.py@@ -730,6 +730,8 @@ async def test_duration_log_reads_unknown_for_a_different_runs_context(duration_logs = [e for e in logs if e.get("event") == "Workflow run duration metrics"]assert len(duration_logs) == 1assert duration_logs[0]["task_v3_ab_arm"] == "unknown"+ # Null, not false: a code-mode run whose attribution was lost must not read as an agent-mode run.+ assert duration_logs[0]["would_be_code"] is None# And no route_reason, which is why the single-log arm read documented in# cloud_docs/feature-flags/task-v3-billing-tier-rollout.md cannot see this population at all: a# route_reason filter drops these rows from both arms, so they have to be counted and@@ -822,8 +824,18 @@ _V3_DEFAULT_CUTOFF = datetime(2026, 9, 15, tzinfo=UTC)"billing_tier": None,},),+ (+ None,+ {WORKFLOW_TASK_V3_AB_FLAG: True},+ {"ineligibility_reason": None, "billing_tier": BillingTier.ENTERPRISE, "would_be_code": True},+ {+ "task_v3_ab_arm": "treatment",+ "route_reason": WorkflowBlockEngineRouteReason.code_bucket_treatment,+ "would_be_code": True,+ },+ ),],- ids=["bucketed-treatment", "bucketed-control", "new-workflow-default", "ineligible"],+ ids=["bucketed-treatment", "bucketed-control", "new-workflow-default", "ineligible", "code-mode-treatment"],)async def test_duration_log_carries_the_tier_and_route_reason_the_run_was_bucketed_on(monkeypatch: pytest.MonkeyPatch,