Skip to content

Automate browser based workflows with AI

Flip the Task V3 A/B coin before the code-mode decision; v3 runs never load or generate scripts (SKY-18016) (#8857)

pedrohsdbb0a79afcec6e
diff --git a/skyvern/core/script_generations/transform_workflow_run.py b/skyvern/core/script_generations/transform_workflow_run.py
index eec2589f5..413933898 100644
--- a/skyvern/core/script_generations/transform_workflow_run.py
+++ b/skyvern/core/script_generations/transform_workflow_run.py
@@ -6,6 +6,7 @@ import structlog
from skyvern.core.script_generations.constants import SCRIPT_TASK_BLOCKS
from skyvern.forge import app
+from skyvern.schemas.run_enums import RunEngine
from skyvern.schemas.workflows import BlockType
from skyvern.services import workflow_service
from skyvern.webeye.actions.action_types import ActionType
@@ -14,6 +15,10 @@ from skyvern.webeye.actions.actions import Action, reasoning_is_turn_scoped
LOG = structlog.get_logger(__name__)
+class TaskV3RunNotScriptableError(Exception):
+ """Task V3 actions carry no element data, so a script built from them has dead selectors."""
+
+
@dataclass
class CodeGenInput:
file_name: str
@@ -258,6 +263,10 @@ async def transform_workflow_run_to_code_gen_input(workflow_run_id: str, organiz
workflow_run_id=workflow_run_id, organization_id=organization_id
)
workflow_run_blocks.sort(key=lambda x: x.created_at)
+ # Checked on the same snapshot the actions are read through: a block row records its engine before any of
+ # its actions exist, so an in-flight mint cannot read a v3 task's actions without also reading its v3 row.
+ if any(block.engine == RunEngine.skyvern_v3 for block in workflow_run_blocks):
+ raise TaskV3RunNotScriptableError(workflow_run_id)
# Create mapping from definition blocks by label for quick lookup
workflow_run_blocks_by_label = {block.label: block for block in workflow_run_blocks if block.label}
diff --git a/skyvern/forge/agent.py b/skyvern/forge/agent.py
index 411b16fad..0cf8f184a 100644
--- a/skyvern/forge/agent.py
+++ b/skyvern/forge/agent.py
@@ -3672,6 +3672,7 @@ class ForgeAgent:
)
return step, detailed_output, None
+ dispatched_as_v3 = engine == RunEngine.skyvern_v3
# A bare task always qualifies; a workflow block must be an allowed type.
task_block_supports_v3 = task_block is None or _task_block_supports_v3(task_block)
if engine == RunEngine.skyvern_v3 and task_block is not None and not task_block_supports_v3:
@@ -3697,7 +3698,7 @@ class ForgeAgent:
"DISABLE_TASK_V3 could not be evaluated; falling back to the step engine",
task_id=task.task_id,
workflow_run_id=task.workflow_run_id,
- # The persisted run type and arm still read v3; cohort reads exclude these by this line.
+ # The arm still reads v3; cohort reads exclude these by this line.
route_reason="flag_error",
exc_info=True,
)
@@ -3745,6 +3746,23 @@ class ForgeAgent:
)
return step, detailed_output, None
+ if dispatched_as_v3 and task.workflow_run_id:
+ # The block row was labeled v3 before dispatch, and script generation skips a run with a
+ # v3 row, so a fallback left labeled v3 would lose a script the run was asked to generate.
+ try:
+ await app.DATABASE.observer.set_workflow_run_block_engine_by_task_id(
+ task.task_id, RunEngine.skyvern_v1, organization_id=task.organization_id
+ )
+ except Exception:
+ LOG.warning(
+ "Could not record the step-engine fallback on the block row",
+ task_id=task.task_id,
+ workflow_run_id=task.workflow_run_id,
+ exc_info=True,
+ )
+ # Later steps recurse with this engine; pinning v1 keeps them off the gate and this write.
+ engine = RunEngine.skyvern_v1
+
if page := await browser_state.get_working_page():
await self.register_async_operations(organization, task, page)
diff --git a/skyvern/forge/sdk/db/repositories/observer.py b/skyvern/forge/sdk/db/repositories/observer.py
index dd80e7f9d..b4dcbaaf4 100644
--- a/skyvern/forge/sdk/db/repositories/observer.py
+++ b/skyvern/forge/sdk/db/repositories/observer.py
@@ -713,6 +713,41 @@ class ObserverRepository(BaseRepository):
).first()
return RunEngine(engine) if engine else None
+ @db_operation("set_workflow_run_block_engine_by_task_id")
+ async def set_workflow_run_block_engine_by_task_id(
+ self,
+ task_id: str,
+ engine: RunEngine,
+ organization_id: str | None = None,
+ ) -> None:
+ async with self.Session() as session:
+ await session.execute(
+ update(WorkflowRunBlockModel)
+ .where(WorkflowRunBlockModel.task_id == task_id)
+ .where(WorkflowRunBlockModel.organization_id == organization_id)
+ .values(engine=engine.value)
+ )
+ await session.commit()
+
+ @db_operation("workflow_run_has_block_on_engine")
+ async def workflow_run_has_block_on_engine(
+ self,
+ workflow_run_id: str,
+ engine: RunEngine,
+ organization_id: str | None = None,
+ ) -> bool:
+ async with self.Session() as session:
+ found = (
+ await session.scalars(
+ select(WorkflowRunBlockModel.workflow_run_block_id)
+ .filter_by(workflow_run_id=workflow_run_id)
+ .filter_by(organization_id=organization_id)
+ .filter_by(engine=engine.value)
+ .limit(1)
+ )
+ ).first()
+ return found is not None
+
@db_operation("get_workflow_run_blocks")
async def get_workflow_run_blocks(
self,
diff --git a/skyvern/forge/sdk/experimentation/workflow_block_engine.py b/skyvern/forge/sdk/experimentation/workflow_block_engine.py
index 1629299b7..a0b1ee059 100644
--- a/skyvern/forge/sdk/experimentation/workflow_block_engine.py
+++ b/skyvern/forge/sdk/experimentation/workflow_block_engine.py
@@ -23,6 +23,7 @@ if TYPE_CHECKING:
LOG = structlog.get_logger()
WORKFLOW_TASK_V3_AB_FLAG = "WORKFLOW_TASK_V3_AB"
+WOULD_BE_CODE_PROPERTY = "would_be_code"
DISABLE_TASK_V3_FLAG = "DISABLE_TASK_V3"
# Trigger kinds whose run executes a workflow the platform minted for that one request rather than
@@ -45,6 +46,10 @@ class WorkflowBlockEngineRouteReason(StrEnum):
flag_bucket_treatment = "flag_bucket_treatment"
flag_bucket_control = "flag_bucket_control"
+ # The same buckets for a run that intended code mode: its control runs cached code or the agent the
+ # code path falls back to, so it is a different comparison from the v3-vs-v1 cells above.
+ code_bucket_treatment = "code_bucket_treatment"
+ code_bucket_control = "code_bucket_control"
new_self_serve_workflow_default = "new_self_serve_workflow_default"
# A workflow born at or after TASK_V3_CHOSEN_ENGINE_CUTOFF, never randomized: at least one block
# left its engine unset and runs on v3 (``new_workflow_v3_default``), or every block's engine was
@@ -90,6 +95,9 @@ class WorkflowBlockEngineArmDecision:
billing_tier: BillingTier | None = None
# An explicit skyvern_v1 on a block is a pin, not a routable default (see Block.resolve_engine).
honors_chosen_engine: bool = False
+ # The run intended code mode when the arm was resolved; control keeps that path. None when no arm was
+ # resolved, so a run whose attribution was lost never reads as an agent-mode run.
+ would_be_code: bool | None = None
NO_ARM_DECISION = WorkflowBlockEngineArmDecision()
@@ -202,6 +210,7 @@ async def _ab_flag_puts_run_in_treatment(
organization_id: str | None,
workflow_permanent_id: str | None,
billing_tier: BillingTier,
+ would_be_code: bool,
) -> bool | None:
"""Whether the percentage knob buckets this run into treatment, or ``None`` if it never answered.
@@ -221,6 +230,9 @@ async def _ab_flag_puts_run_in_treatment(
"organization_id": organization_id,
"workflow_permanent_id": workflow_permanent_id or "not_workflow",
BILLING_TIER_PROPERTY: billing_tier.value,
+ # Sent for reads only. A condition on it would be inconclusive, so None (control), for any
+ # evaluation that omits it: posthog's local match raises on a property it was not given.
+ WOULD_BE_CODE_PROPERTY: "true" if would_be_code else "false",
},
)
@@ -236,6 +248,7 @@ async def resolve_workflow_block_engine_arm(
ineligibility_reason: V3AbIneligibleReason | None,
takes_default_engine: bool | None,
pinned_v1_blocks: int = 0,
+ would_be_code: bool = False,
) -> None:
"""Resolve the workflow-block engine A/B once at execution start and pin the arm on the context.
@@ -266,6 +279,12 @@ async def resolve_workflow_block_engine_arm(
``pinned_v1_blocks`` counts blocks a person pinned to skyvern-1.0 (see ``pinned_v1_block_count``);
eligibility already keeps such a run out of the A/B, and this labels it ``pinned_v1_engine``.
+
+ ``would_be_code`` says the run intended code mode (a cached script, the no-script-yet code_generation
+ mode, or a code-rollout upgrade), whether or not a script ends up executing. An explicit ``code_gen``
+ request is not bucketed: it is ineligible. Only the percentage knob may move
+ such a run: a treatment bucket runs it as a v3 agent without its script (``code_mode_displaced``),
+ while every other route leaves it on the code path it had.
"""
if context.workflow_block_engine_resolved_run_id == workflow_run_id:
return
@@ -331,7 +350,8 @@ async def resolve_workflow_block_engine_arm(
# The tier, the status and the trigger are checked first, so anything that
# cannot use the rule pays for no workflow read.
takes_new_workflow_default = (
- billing_tier == BillingTier.SELF_SERVE
+ not would_be_code
+ and billing_tier == BillingTier.SELF_SERVE
and _is_kept_workflow_run(workflow_status, trigger_type)
and await born_at_or_after(settings.TASK_V3_DEFAULT_ENGINE_WORKFLOW_CUTOFF)
)
@@ -347,17 +367,26 @@ async def resolve_workflow_block_engine_arm(
organization_id=organization_id,
workflow_permanent_id=workflow_permanent_id,
billing_tier=billing_tier,
+ would_be_code=would_be_code,
)
if in_treatment:
override = RunEngine.skyvern_v3
- route_reason = WorkflowBlockEngineRouteReason.flag_bucket_treatment
+ route_reason = (
+ WorkflowBlockEngineRouteReason.code_bucket_treatment
+ if would_be_code
+ else WorkflowBlockEngineRouteReason.flag_bucket_treatment
+ )
elif in_treatment is None:
# The flag never answered, so this run is on control without having been
# randomized. Labelled apart from the bucket so the control cell every
# per-arm read builds out of flag_bucket_control stays a randomized one.
route_reason = WorkflowBlockEngineRouteReason.flag_undefined
else:
- route_reason = WorkflowBlockEngineRouteReason.flag_bucket_control
+ route_reason = (
+ WorkflowBlockEngineRouteReason.code_bucket_control
+ if would_be_code
+ else WorkflowBlockEngineRouteReason.flag_bucket_control
+ )
except Exception:
LOG.warning(
"Failed to resolve the workflow-block engine arm; using control",
@@ -371,6 +400,7 @@ async def resolve_workflow_block_engine_arm(
route_reason=route_reason,
billing_tier=billing_tier,
honors_chosen_engine=honors_chosen_engine,
+ would_be_code=would_be_code,
)
context.workflow_block_engine_override = override
context.workflow_block_engine_arm_decision = decision
@@ -390,9 +420,22 @@ async def resolve_workflow_block_engine_arm(
# doubles as "nobody looked".
billing_tier=engine_arm_log_value(billing_tier),
pinned_v1_blocks=pinned_v1_blocks,
+ would_be_code=would_be_code,
)
+def code_mode_displaced(workflow_run_id: str | None) -> bool:
+ """Whether the A/B bucketed this run, headed for a cached script, into treatment: it then runs every
+ block as a v3 agent instead of the script."""
+ if not workflow_run_id:
+ return False
+ context = skyvern_context.current()
+ if context is None or context.workflow_block_engine_resolved_run_id != workflow_run_id:
+ return False
+ decision = context.workflow_block_engine_arm_decision
+ return bool(decision and decision.route_reason == WorkflowBlockEngineRouteReason.code_bucket_treatment)
+
+
def workflow_block_engine_override(workflow_run_id: str | None) -> RunEngine | None:
"""The engine the A/B pinned for this run, or None when the run is control or unresolved.
diff --git a/skyvern/forge/sdk/workflow/models/block.py b/skyvern/forge/sdk/workflow/models/block.py
index b37c62640..5b153271d 100644
--- a/skyvern/forge/sdk/workflow/models/block.py
+++ b/skyvern/forge/sdk/workflow/models/block.py
@@ -2078,8 +2078,8 @@ class BaseTaskBlock(Block):
def resolve_engine(self, workflow_run_id: str | None) -> RunEngine:
"""The engine this block dispatches to, after the per-run A/B.
- Both the persisted workflow_run_blocks.engine and the execute_step dispatch read this, so
- the recorded engine cannot disagree with the one that ran. A block pinned to a non-default
+ Both the persisted workflow_run_blocks.engine and the execute_step dispatch read this; when the
+ dispatch falls back to the step engine, execute_step corrects the row to the engine that ran. A block pinned to a non-default
engine is honored as-authored, and a block the eligibility check never saw is left alone;
neither is ever rerouted. An unset engine routes like skyvern_v1, except in a run that honors the
chosen engine or on a block whose skyvern_v1 a person pinned.
@@ -2462,6 +2462,8 @@ class BaseTaskBlock(Block):
workflow_run_block_id=workflow_run_block_id,
task_id=task.task_id,
organization_id=organization_id,
+ # A retry reuses this row, which a step-engine fallback on the previous attempt relabeled.
+ engine=self.resolve_engine(workflow_run_id).value,
)
current_running_task = task
organization = await app.DATABASE.organizations.get_organization(
@@ -8184,6 +8186,8 @@ async def wrapper({default_args}):
attempt_number=workflow_run_context.attempt_number,
label="Self-heal recovery",
block_type=BlockType.TASK,
+ # Dispatched on v3 below; script generation reads this column to keep v3 actions out.
+ engine=RunEngine.skyvern_v3,
)
recovery_block_id = recovery_block.workflow_run_block_id
@@ -19824,13 +19828,14 @@ class V3AbIneligibleReason(StrEnum):
eligible" but not "why not."
"""
- script_run = "script_run"
pinned_engine = "pinned_engine"
+ # An explicit request to generate code from this run; v3 actions cannot produce a usable script.
+ code_generation_requested = "code_generation_requested"
unsupported_block = "unsupported_block"
no_reroutable_blocks = "no_reroutable_blocks"
-def v3_ab_ineligibility_reason(blocks: list[BlockTypeVar], *, is_script_run: bool) -> V3AbIneligibleReason | None:
+def v3_ab_ineligibility_reason(blocks: list[BlockTypeVar]) -> V3AbIneligibleReason | None:
"""Why a whole workflow run may not be rerouted onto v3 by the A/B, or None if it may.
Eligibility is a property of the RUN, not of a block: a run whose blocks disagreed about the
@@ -19847,12 +19852,9 @@ def v3_ab_ineligibility_reason(blocks: list[BlockTypeVar], *, is_script_run: boo
mixed-arm run this predicate exists to prevent -- and a partial re-run is not comparable to a
full run in the cohort anyway.
- Script runs are excluded: their blocks execute as cached code and never reach engine dispatch,
- so treatment would land only on the ai_fallback subset -- the blocks that already failed cached
- execution.
+ A run headed for a cached script is eligible: treatment runs it as a v3 agent without the script
+ (see ``code_mode_displaced``), so the arm covers every block rather than only the ai_fallback ones.
"""
- if is_script_run:
- return V3AbIneligibleReason.script_run
reroutable_blocks = 0
for block in blocks:
if isinstance(block, ConditionalBlock):
@@ -19919,9 +19921,9 @@ def takes_default_engine(blocks: list[BlockTypeVar]) -> bool | None:
return False if has_engine_block else None
-def run_is_eligible_for_v3_ab(blocks: list[BlockTypeVar], *, is_script_run: bool) -> bool:
+def run_is_eligible_for_v3_ab(blocks: list[BlockTypeVar]) -> bool:
"""Whether a whole workflow run may be rerouted onto v3 by the A/B; see v3_ab_ineligibility_reason."""
- return v3_ab_ineligibility_reason(blocks, is_script_run=is_script_run) is None
+ return v3_ab_ineligibility_reason(blocks) is None
def get_all_blocks(blocks: list[BlockTypeVar]) -> list[BlockTypeVar]:
diff --git a/skyvern/forge/sdk/workflow/service.py b/skyvern/forge/sdk/workflow/service.py
index 7f62bfa26..540ac8775 100644
--- a/skyvern/forge/sdk/workflow/service.py
+++ b/skyvern/forge/sdk/workflow/service.py
@@ -132,6 +132,7 @@ from skyvern.forge.sdk.experimentation.transient_ui_capture import resolve_trans
from skyvern.forge.sdk.experimentation.workflow_block_engine import (
ARM_ATTRIBUTION_LOST,
WorkflowBlockEngineArmAttribution,
+ code_mode_displaced,
engine_arm_log_value,
resolve_workflow_block_engine_arm,
resolved_workflow_block_engine_arm_attribution,
@@ -7706,6 +7707,7 @@ class WorkflowService:
if (
is_adaptive_caching(workflow, workflow_run)
and is_script_execution
+ and not await self._ran_a_block_on_v3(workflow_run)
and pre_finally_status
not in (
WorkflowRunStatus.canceled,
@@ -8158,7 +8160,11 @@ class WorkflowService:
# share an arm. Covers the DAG executor too, which this method delegates to.
current_context = skyvern_context.current()
if current_context:
- ineligibility_reason = v3_ab_ineligibility_reason(all_blocks, is_script_run=is_script_run)
+ ineligibility_reason = (
+ V3AbIneligibleReason.code_generation_requested
+ if workflow_run.code_gen
+ else v3_ab_ineligibility_reason(all_blocks)
+ )
await resolve_workflow_block_engine_arm(
current_context,
workflow_run_id=workflow_run_id,
@@ -8167,13 +8173,16 @@ class WorkflowService:
workflow_status=workflow.status,
trigger_type=workflow_run.trigger_type,
ineligibility_reason=ineligibility_reason,
- takes_default_engine=takes_default_engine(all_blocks),
+ # A code_gen request has no engine to choose: the chosen-engine rule would put it on v3,
+ # whose actions cannot yield the script it asked for.
+ takes_default_engine=None if workflow_run.code_gen else takes_default_engine(all_blocks),
# Counted only when the pin is what left the A/B, so pinned_v1_engine never labels another reason.
pinned_v1_blocks=(
pinned_v1_block_count(all_blocks)
if ineligibility_reason == V3AbIneligibleReason.pinned_engine
else 0
),
+ would_be_code=is_script_run,
)
else:
LOG.warning(
@@ -8181,6 +8190,18 @@ class WorkflowService:
workflow_run_id=workflow_run_id,
)
+ if is_script_run and code_mode_displaced(workflow_run_id):
+ LOG.info(
+ "A/B treatment runs this code-mode run on v3 as an agent, without its script",
+ workflow_run_id=workflow_run_id,
+ script_id=script.script_id if script else None,
+ )
+ script = None
+ script_is_pinned = False
+ is_script_run = False
+ if current_context:
+ current_context.generate_script = False
+
# A cached script can be superseded (e.g. a stale pin replaced by a platform static
# script) — drop it here so the ensure_static_script path below re-creates the pin.
if script and is_script_run:
@@ -8665,6 +8686,24 @@ class WorkflowService:
name=f"script_gen_{workflow_run.workflow_run_id}",
)
+ async def _ran_a_block_on_v3(self, workflow_run: WorkflowRun) -> bool:
+ """Task V3 records no element data on its actions, so a script generated from a run that used it
+ carries dead selectors, and later code-mode runs of the workflow would load it."""
+ try:
+ return await app.DATABASE.observer.workflow_run_has_block_on_engine(
+ workflow_run_id=workflow_run.workflow_run_id,
+ engine=RunEngine.skyvern_v3,
+ organization_id=workflow_run.organization_id,
+ )
+ except Exception:
+ # Unknown counts as v3: a skipped script is regenerated by the next v1 run, a dead one is loaded.
+ LOG.warning(
+ "Could not read the run's block engines; skipping script work",
+ workflow_run_id=workflow_run.workflow_run_id,
+ exc_info=True,
+ )
+ return True
+
async def _do_generate_pending_script(
self,
workflow: Workflow,
@@ -8672,6 +8711,8 @@ class WorkflowService:
) -> None:
"""Fire-and-forget wrapper for pending script generation with error handling."""
try:
+ if await self._ran_a_block_on_v3(workflow_run):
+ return
await workflow_script_service.generate_or_update_pending_workflow_script(
workflow_run=workflow_run,
workflow=workflow,
@@ -13437,6 +13478,7 @@ class WorkflowService:
# run that was never randomized. Per-arm reads filter on route_reason, and the tier
# is what the non-enterprise ramp steps on.
route_reason=engine_arm_log_value(arm_decision.route_reason),
+ would_be_code=arm_decision.would_be_code,
billing_tier=engine_arm_log_value(arm_decision.billing_tier),
)
# Run minutes measure compute. A run finalized without ever reaching
@@ -17448,6 +17490,14 @@ class WorkflowService:
# request is not made
return
+ if await self._ran_a_block_on_v3(workflow_run):
+ LOG.info(
+ "Skipping script generation: this run executed a block on Task V3",
+ workflow_run_id=workflow_run.workflow_run_id,
+ workflow_permanent_id=workflow.workflow_permanent_id,
+ )
+ return
+
existing_script, rendered_cache_key_value, _is_pinned = await workflow_script_service.get_workflow_script(
workflow,
workflow_run,
diff --git a/skyvern/services/script_service.py b/skyvern/services/script_service.py
index e4af6973f..10aa04521 100644
--- a/skyvern/services/script_service.py
+++ b/skyvern/services/script_service.py
@@ -1745,6 +1745,15 @@ async def _fallback_to_ai_run(
complete_verification=complete_verification,
include_action_history_in_verification=include_action_history_in_verification,
)
+ if workflow_run_block_id and engine == RunEngine.skyvern_v3:
+ # The row was created for cached code with no engine; script generation and the reviewer
+ # read this column to keep a v3 run's actions out of the workflow's script. Not swallowed:
+ # a v3 fallback whose row still reads v1 could be minted into a dead-selector script.
+ await app.DATABASE.observer.update_workflow_run_block(
+ workflow_run_block_id=workflow_run_block_id,
+ organization_id=organization_id,
+ engine=engine.value,
+ )
await app.agent.execute_step(
organization=organization,
task=task,
diff --git a/skyvern/services/workflow_script_service.py b/skyvern/services/workflow_script_service.py
index 8a0edee94..f62366c1f 100644
--- a/skyvern/services/workflow_script_service.py
+++ b/skyvern/services/workflow_script_service.py
@@ -18,7 +18,10 @@ from skyvern.core.script_generations.generate_script import (
ScriptBlockSource,
generate_workflow_script_python_code,
)
-from skyvern.core.script_generations.transform_workflow_run import transform_workflow_run_to_code_gen_input
+from skyvern.core.script_generations.transform_workflow_run import (
+ TaskV3RunNotScriptableError,
+ transform_workflow_run_to_code_gen_input,
+)
from skyvern.forge import app
from skyvern.forge.sdk.cache.factory import CacheFactory
from skyvern.forge.sdk.core import skyvern_context
@@ -1053,6 +1056,13 @@ async def generate_workflow_script(
use_semantic_selectors=adaptive,
adaptive_caching=adaptive,
)
+ except TaskV3RunNotScriptableError:
+ LOG.info(
+ "Skipping script generation: this run executed a block on Task V3",
+ workflow_run_id=workflow_run.workflow_run_id,
+ workflow_permanent_id=workflow.workflow_permanent_id,
+ )
+ return
except Exception:
generation_duration_ms = (time.monotonic() - generation_start) * 1000
LOG.error(
diff --git a/tests/unit/_workflow_block_engine_fakes.py b/tests/unit/_workflow_block_engine_fakes.py
index 1e3b33588..28e331b56 100644
--- a/tests/unit/_workflow_block_engine_fakes.py
+++ b/tests/unit/_workflow_block_engine_fakes.py
@@ -119,6 +119,7 @@ async def resolve_arm(
workflow_status: WorkflowStatus = WorkflowStatus.published,
trigger_type: WorkflowRunTriggerType | None = WorkflowRunTriggerType.api,
takes_default_engine: bool | None = True,
+ would_be_code: bool = False,
) -> Resolution:
async def read_birth_timestamp(workflow_permanent_id: str, organization_id: str) -> datetime | None:
if first_version_error is not None:
@@ -163,6 +164,7 @@ async def resolve_arm(
trigger_type=trigger_type,
ineligibility_reason=ineligibility_reason,
takes_default_engine=takes_default_engine,
+ would_be_code=would_be_code,
)
logged = dict(mock_log.info.call_args.kwargs) if mock_log.info.call_args else {}
return Resolution(log=logged, birth_reads=birth_reads, warnings=list(mock_log.warning.call_args_list))
diff --git a/tests/unit/test_agent_task_v3.py b/tests/unit/test_agent_task_v3.py
index 8c02b9814..4c0940f24 100644
--- a/tests/unit/test_agent_task_v3.py
+++ b/tests/unit/test_agent_task_v3.py
@@ -1930,6 +1930,7 @@ async def _run_execute_step_gate(
workflow_run: Any = None,
workflow_owned_recovery: bool = False,
recovery_code_progress: CodeProgressRecord | None = None,
+ observer: Any = None,
**task_overrides: Any,
) -> tuple[AsyncMock, AsyncMock]:
"""Drive ForgeAgent.execute_step through the v3 dispatch gate and return (mocked _execute_task_v3,
@@ -1967,6 +1968,10 @@ async def _run_execute_step_gate(
mock_app.DATABASE.tasks.get_task = AsyncMock(return_value=None)
mock_app.DATABASE.tasks.update_task = AsyncMock()
mock_app.DATABASE.workflow_runs.get_workflow_run = AsyncMock(return_value=workflow_run)
+ if observer is not None:
+ mock_app.DATABASE.observer = observer
+ else:
+ mock_app.DATABASE.observer.set_workflow_run_block_engine_by_task_id = AsyncMock()
mock_app.AGENT_FUNCTION.validate_step_execution = AsyncMock()
if experimentation_provider is not None:
mock_app.EXPERIMENTATION_PROVIDER = experimentation_provider
@@ -2124,7 +2129,7 @@ async def test_an_unevaluable_kill_switch_runs_an_explicit_v3_block_on_the_step_
# The engine handed down is the one every later step of this task recurses with, so a v3 pin
# here would re-read the flag next step and could restart the task on v3 mid-way.
assert step_engine_mock.await_args.kwargs["engine"] == agent_module.RunEngine.skyvern_v1
- # The run's persisted type and arm still read v3, so this line is what cohort reads exclude it by.
+ # The run's arm still reads v3, so this line is what cohort reads exclude it by.
assert any(
log.get("route_reason") == "flag_error" and log.get("workflow_run_id") == "wr_task_v3_kill_down" for log in logs
)
diff --git a/tests/unit/test_missing_starter_url.py b/tests/unit/test_missing_starter_url.py
index 844684480..dff345280 100644
--- a/tests/unit/test_missing_starter_url.py
+++ b/tests/unit/test_missing_starter_url.py
@@ -128,6 +128,7 @@ def _mock_block_execute_deps(
yield {
"task": task,
"tasks_db": tasks_db,
+ "observer_db": observer_db,
"agent": agent,
"browser_state": browser_state,
"browser_manager": browser_manager,
diff --git a/tests/unit/test_pending_script_promotion.py b/tests/unit/test_pending_script_promotion.py
index 2d456ca8b..26371ec96 100644
--- a/tests/unit/test_pending_script_promotion.py
+++ b/tests/unit/test_pending_script_promotion.py
@@ -23,12 +23,17 @@ from skyvern.core.script_generations.generate_script import _build_block_stateme
from skyvern.forge import app
from skyvern.forge.sdk.core import skyvern_context
from skyvern.forge.sdk.core.skyvern_context import SkyvernContext
-from skyvern.forge.sdk.workflow.models.block import ExtractionBlock
+from skyvern.forge.sdk.experimentation.providers import BaseExperimentationProvider
+from skyvern.forge.sdk.workflow.models.block import ExtractionBlock, TaskBlock
from skyvern.forge.sdk.workflow.models.parameter import OutputParameter
from skyvern.forge.sdk.workflow.service import WorkflowService
+from skyvern.schemas.run_enums import RunEngine
from skyvern.schemas.scripts import ScriptStatus
from skyvern.schemas.workflows import BlockStatus, BlockType
from skyvern.services import workflow_script_service
+from skyvern.services import workflow_service as workflow_service_module
+from skyvern.webeye.actions.actions import ClickAction
+from tests.unit.test_agent_task_v3 import _make_block, _run_execute_step_gate
def make_workflow(block_types: list[BlockType]) -> SimpleNamespace:
@@ -378,9 +383,40 @@ class TestPendingMintUsesOriginalRevision:
scripts.create_workflow_script.assert_not_awaited()
+def run_blocks_db(*engines: RunEngine | None) -> SimpleNamespace:
+ async def has_block_on_engine(*, workflow_run_id: str, engine: RunEngine, organization_id: str) -> bool:
+ return engine in engines
+
+ return SimpleNamespace(workflow_run_has_block_on_engine=has_block_on_engine)
+
+
+class BlockRowsByTask:
+ """Block-row engines keyed by task, written by the dispatch fallback and read by the mint guard."""
+
+ def __init__(self, **engines: RunEngine) -> None:
+ self.engines = dict(engines)
+
+ async def set_workflow_run_block_engine_by_task_id(
+ self, task_id: str, engine: RunEngine, organization_id: str | None = None
+ ) -> None:
+ self.engines[task_id] = engine
+
+ async def workflow_run_has_block_on_engine(
+ self, *, workflow_run_id: str, engine: RunEngine, organization_id: str | None
+ ) -> bool:
+ return engine in self.engines.values()
+
+
class TestFinalizeReusesPendingScript:
- def _patch_common(self, monkeypatch: pytest.MonkeyPatch, scripts: MagicMock) -> AsyncMock:
- monkeypatch.setattr(app, "DATABASE", SimpleNamespace(scripts=scripts), raising=False)
+ def _patch_common(
+ self,
+ monkeypatch: pytest.MonkeyPatch,
+ scripts: MagicMock,
+ *engines: RunEngine | None,
+ observer: BlockRowsByTask | None = None,
+ ) -> AsyncMock:
+ database = SimpleNamespace(scripts=scripts, observer=observer or run_blocks_db(*engines))
+ monkeypatch.setattr(app, "DATABASE", database, raising=False)
monkeypatch.setattr(app, "ARTIFACT_MANAGER", SimpleNamespace(upload_aiotasks_map={}), raising=False)
monkeypatch.setattr(
workflow_script_service,
@@ -457,3 +493,168 @@ class TestFinalizeReusesPendingScript:
scripts.create_script.assert_awaited_once()
assert generate_mock.await_args.kwargs["script"].script_id == "s_new"
+
+
+class TestNoScriptFromATaskV3Run:
+ """Task V3 actions carry no element data, so a script minted from them has dead selectors, and the
+ workflow's later code-mode runs would load it. Both mint paths must refuse such a run."""
+
+ @pytest.mark.parametrize(
+ ("engines", "mints"),
+ [
+ ((RunEngine.skyvern_v1, RunEngine.skyvern_v1), True),
+ ((RunEngine.skyvern_v1, RunEngine.skyvern_v3), False),
+ ((RunEngine.skyvern_v3,), False),
+ ],
+ )
+ @pytest.mark.asyncio
+ async def test_finalize_mints_only_from_a_run_with_no_v3_block(
+ self, monkeypatch: pytest.MonkeyPatch, engines: tuple[RunEngine, ...], mints: bool
+ ) -> None:
+ scripts = make_scripts_db(pending_row=None)
+ generate_mock = TestFinalizeReusesPendingScript()._patch_common(monkeypatch, scripts, *engines)
+
+ await WorkflowService().generate_script_if_needed(
+ workflow=make_workflow([BlockType.GOTO_URL, BlockType.TASK]),
+ workflow_run=make_workflow_run(),
+ finalize=True,
+ )
+
+ assert scripts.create_script.await_count == int(mints)
+ assert generate_mock.await_count == int(mints)
+
+ @pytest.mark.parametrize(("engine", "mints"), [(RunEngine.skyvern_v1, True), (RunEngine.skyvern_v3, False)])
+ @pytest.mark.asyncio
+ async def test_pending_mint_skips_a_run_with_a_v3_block(
+ self, monkeypatch: pytest.MonkeyPatch, engine: RunEngine, mints: bool
+ ) -> None:
+ monkeypatch.setattr(app, "DATABASE", SimpleNamespace(observer=run_blocks_db(engine)), raising=False)
+ pending_mint = AsyncMock()
+ monkeypatch.setattr(workflow_script_service, "generate_or_update_pending_workflow_script", pending_mint)
+
+ await WorkflowService()._do_generate_pending_script(make_workflow([BlockType.TASK]), make_workflow_run())
+
+ assert pending_mint.await_count == int(mints)
+
+ @pytest.mark.parametrize(("kill_switch_on", "mints"), [(True, True), (False, False)])
+ @pytest.mark.asyncio
+ async def test_a_v3_pinned_block_the_kill_switch_ran_on_v1_still_gets_its_requested_script(
+ self, monkeypatch: pytest.MonkeyPatch, kill_switch_on: bool, mints: bool
+ ) -> None:
+ workflow_run = make_workflow_run()
+ workflow_run.code_gen = True
+ # execute_safe labels the row with the resolved engine before dispatch.
+ rows = BlockRowsByTask(**{"task-123": RunEngine.skyvern_v3})
+ provider = MagicMock(spec=BaseExperimentationProvider)
+ provider.resolve_feature_flag_strict = AsyncMock(return_value=kill_switch_on)
+
+ await _run_execute_step_gate(
+ engine=RunEngine.skyvern_v3,
+ task_block=_make_block(TaskBlock, label="pinned", engine=RunEngine.skyvern_v3),
+ experimentation_provider=provider,
+ observer=rows,
+ workflow_run_id=workflow_run.workflow_run_id,
+ )
+ scripts = make_scripts_db(pending_row=None)
+ generate_mock = TestFinalizeReusesPendingScript()._patch_common(monkeypatch, scripts, observer=rows)
+
+ await WorkflowService().generate_script_if_needed(
+ workflow=make_workflow([BlockType.GOTO_URL, BlockType.TASK]),
+ workflow_run=workflow_run,
+ )
+
+ assert generate_mock.await_count == int(mints)
+
+ @pytest.mark.asyncio
+ async def test_a_non_v3_dispatch_keeps_its_row_and_engine(self) -> None:
+ rows = BlockRowsByTask(**{"task-123": RunEngine.openai_cua})
+
+ _v3_mock, step_engine_mock = await _run_execute_step_gate(
+ engine=RunEngine.openai_cua,
+ task_block=_make_block(TaskBlock, label="cua", engine=RunEngine.openai_cua),
+ observer=rows,
+ workflow_run_id=make_workflow_run().workflow_run_id,
+ )
+
+ assert rows.engines == {"task-123": RunEngine.openai_cua}
+ assert step_engine_mock.await_args.kwargs["engine"] == RunEngine.openai_cua
+
+ @pytest.mark.parametrize(
+ ("second_block_engine", "mints"), [(RunEngine.skyvern_v1, True), (RunEngine.skyvern_v3, False)]
+ )
+ @pytest.mark.asyncio
+ async def test_an_in_flight_pending_mint_whose_snapshot_holds_v3_actions_writes_nothing(
+ self, monkeypatch: pytest.MonkeyPatch, second_block_engine: RunEngine, mints: bool
+ ) -> None:
+ # The mint passed its v3 check before the second block's row existed; by the snapshot that block's
+ # actions are stored, so only the snapshot itself can refuse them.
+ def run_block(label: str, task_id: str, engine: RunEngine, created_at: int) -> SimpleNamespace:
+ return SimpleNamespace(
+ workflow_run_block_id=f"wrb_{label}",
+ parent_workflow_run_block_id=None,
+ block_type=BlockType.TASK,
+ label=label,
+ task_id=task_id,
+ status="completed",
+ output={},
+ created_at=created_at,
+ engine=engine,
+ )
+
+ definition_blocks = [
+ SimpleNamespace(label=label, block_type=BlockType.TASK, model_dump=lambda label=label: {"label": label})
+ for label in ("cached", "pinned")
+ ]
+ workflow = SimpleNamespace(
+ workflow_id="w_1",
+ workflow_permanent_id="wpid_1",
+ organization_id="o_1",
+ title="wf",
+ workflow_definition=SimpleNamespace(blocks=definition_blocks),
+ model_dump=lambda: {"workflow_id": "w_1"},
+ )
+ run_request = SimpleNamespace(workflow_id="wpid_1", model_dump=lambda: {"workflow_id": "wpid_1"})
+ tasks = [
+ SimpleNamespace(task_id=t, model_dump=lambda t=t: {"task_id": t}) for t in ("tsk_cached", "tsk_pinned")
+ ]
+ actions = [
+ ClickAction(action_id=f"a_{t}", task_id=t, element_id=f"el_{t}") for t in ("tsk_pinned", "tsk_cached")
+ ]
+ database = SimpleNamespace(
+ observer=SimpleNamespace(
+ get_workflow_run_blocks=AsyncMock(
+ return_value=[
+ run_block("cached", "tsk_cached", RunEngine.skyvern_v1, 1),
+ run_block("pinned", "tsk_pinned", second_block_engine, 2),
+ ]
+ )
+ ),
+ tasks=SimpleNamespace(
+ get_tasks_by_ids=AsyncMock(return_value=tasks), get_tasks_actions=AsyncMock(return_value=actions)
+ ),
+ )
+ monkeypatch.setattr(app, "DATABASE", database, raising=False)
+ monkeypatch.setattr(
+ app,
+ "WORKFLOW_SERVICE",
+ SimpleNamespace(get_workflow_by_permanent_id=AsyncMock(return_value=workflow)),
+ raising=False,
+ )
+ monkeypatch.setattr(
+ workflow_service_module,
+ "get_workflow_run_response",
+ AsyncMock(return_value=SimpleNamespace(run_request=run_request)),
+ )
+ monkeypatch.setattr(workflow_script_service, "is_adaptive_caching", lambda *_: False)
+ codegen = AsyncMock(side_effect=RuntimeError("stop after the generation input is accepted"))
+ monkeypatch.setattr(workflow_script_service, "generate_workflow_script_python_code", codegen)
+
+ await workflow_script_service.generate_workflow_script(
+ workflow_run=make_workflow_run(),
+ workflow=workflow,
+ script=SimpleNamespace(script_id="s_1", script_revision_id="sr_1", version=1),
+ rendered_cache_key_value="default:site",
+ pending=True,
+ )
+
+ assert codegen.await_count == int(mints)
diff --git a/tests/unit/test_script_service_fallback_engine.py b/tests/unit/test_script_service_fallback_engine.py
index b7bbdf4d8..1bc0be89a 100644
--- a/tests/unit/test_script_service_fallback_engine.py
+++ b/tests/unit/test_script_service_fallback_engine.py
@@ -266,7 +266,7 @@ async def test_fallback_of_an_unset_engine_block_follows_the_chosen_engine_cutof
context,
FakeExperimentationProvider(),
workflow_run_id="wr_test",
- ineligibility_reason=v3_ab_ineligibility_reason(blocks, is_script_run=True),
+ ineligibility_reason=v3_ab_ineligibility_reason(blocks),
takes_default_engine=takes_default_engine(blocks),
first_version_created_at=born_at,
)
@@ -333,7 +333,7 @@ async def test_an_uncached_script_block_runs_where_its_stored_block_would_only_p
context,
FakeExperimentationProvider(),
workflow_run_id="wr_test",
- ineligibility_reason=v3_ab_ineligibility_reason(workflow.workflow_definition.blocks, is_script_run=True),
+ ineligibility_reason=v3_ab_ineligibility_reason(workflow.workflow_definition.blocks),
takes_default_engine=takes_default_engine(workflow.workflow_definition.blocks),
first_version_created_at=born_at,
)
@@ -546,3 +546,62 @@ async def test_fallback_episode_excludes_decision_row_from_agent_action_count()
update_episode.await_args.kwargs["agent_actions"]["failure_reason"]
== script_service.VERIFIER_SWAP_FAILURE_REASON
)
+
+
+class _DispatchReached(Exception):
+ pass
+
+
+ ("block_engine", "recorded"), [(RunEngine.skyvern_v3, ["skyvern-3.0"]), (RunEngine.skyvern_v1, [])]
+)
+async def test_a_v3_fallback_records_its_engine_on_the_cached_blocks_row(
+ block_engine: RunEngine, recorded: list[str]
+) -> None:
+ # The row was created for cached code with no engine. Script generation and the reviewer read this
+ # column to keep a v3 run's selector-less actions out of the workflow's script.
+ workflow = _make_workflow([_make_task_block("my_block", engine=block_engine)])
+ app = _make_app(workflow)
+ app.DATABASE.observer.update_workflow_run_block = AsyncMock()
+ # The engine is written before dispatch; stop there instead of faking everything after it.
+ app.agent.execute_step = AsyncMock(side_effect=_DispatchReached())
+ with (
+ patch(f"{MODULE}.app", app),
+ patch(f"{MODULE}.skyvern_context.current", return_value=_make_context()),
+ pytest.raises(_DispatchReached),
+ ):
+ await script_service._fallback_to_ai_run(
+ block_type=BlockType.NAVIGATION,
+ cache_key="my_block",
+ prompt="do the thing",
+ workflow_run_block_id="wrb_cached",
+ )
+
+ engines = [
+ call.kwargs["engine"]
+ for call in app.DATABASE.observer.update_workflow_run_block.call_args_list
+ if "engine" in call.kwargs
+ ]
+ assert engines == recorded
+
+
+async def test_a_v3_fallback_that_cannot_record_its_engine_does_not_run() -> None:
+ # Running anyway would leave a v3 run whose rows read v1, which script generation would then mint.
+ workflow = _make_workflow([_make_task_block("my_block", engine=RunEngine.skyvern_v3)])
+ app = _make_app(workflow)
+ app.DATABASE.observer.update_workflow_run_block = AsyncMock(side_effect=RuntimeError("db unavailable"))
+ with (
+ patch(f"{MODULE}.app", app),
+ patch(f"{MODULE}.skyvern_context.current", return_value=_make_context()),
+ pytest.raises(RuntimeError),
+ ):
+ await script_service._fallback_to_ai_run(
+ block_type=BlockType.NAVIGATION,
+ cache_key="my_block",
+ prompt="do the thing",
+ workflow_run_block_id="wrb_cached",
+ )
+
+ app.agent.execute_step.assert_not_awaited()
diff --git a/tests/unit/test_workflow_block_engine.py b/tests/unit/test_workflow_block_engine.py
index b6abfc4c6..adfe4fd69 100644
--- a/tests/unit/test_workflow_block_engine.py
+++ b/tests/unit/test_workflow_block_engine.py
@@ -25,12 +25,15 @@ from skyvern.forge.sdk.experimentation.providers import BaseExperimentationProvi
from skyvern.forge.sdk.experimentation.workflow_block_engine import (
DISABLE_TASK_V3_FLAG,
WORKFLOW_TASK_V3_AB_FLAG,
+ WOULD_BE_CODE_PROPERTY,
WorkflowBlockEngineRouteReason,
_as_utc,
+ code_mode_displaced,
effective_default_engine,
resolve_workflow_block_engine_arm,
workflow_block_engine_override,
)
+from skyvern.forge.sdk.schemas.tasks import TaskStatus
from skyvern.forge.sdk.workflow.models.block import (
ActionBlock,
BaseTaskBlock,
@@ -121,7 +124,7 @@ def test_mixed_eligibility_run_download_block_is_eligible(scoped_context: Skyver
download_block = _make_block(ActionBlock, label="dl", complete_on_download=True)
blocks: list[BaseTaskBlock] = [eligible_1, eligible_2, download_block]
- assert run_is_eligible_for_v3_ab(blocks, is_script_run=False) is True
+ assert run_is_eligible_for_v3_ab(blocks) is True
def test_mixed_eligibility_run_file_download_block_is_eligible(scoped_context: SkyvernContext) -> None:
@@ -130,7 +133,7 @@ def test_mixed_eligibility_run_file_download_block_is_eligible(scoped_context: S
file_download_block = _make_block(FileDownloadBlock, label="fd")
blocks: list[BaseTaskBlock] = [eligible_1, eligible_2, file_download_block]
- assert run_is_eligible_for_v3_ab(blocks, is_script_run=False) is True
+ assert run_is_eligible_for_v3_ab(blocks) is True
def test_mixed_eligibility_run_download_gated_validation_block_is_not_eligible(
@@ -143,7 +146,7 @@ def test_mixed_eligibility_run_download_gated_validation_block_is_not_eligible(
download_gated_validation = _make_block(ValidationBlock, label="dlv", complete_on_download=True)
blocks: list[BaseTaskBlock] = [eligible_1, eligible_2, download_gated_validation]
- assert run_is_eligible_for_v3_ab(blocks, is_script_run=False) is False
+ assert run_is_eligible_for_v3_ab(blocks) is False
@pytest.mark.asyncio
@@ -154,7 +157,7 @@ async def test_mixed_eligibility_run_pins_whole_run_to_control(scoped_context: S
ineligible_block.block_type = BlockType.WAIT
blocks: list[BaseTaskBlock] = [eligible_1, eligible_2, ineligible_block]
- assert run_is_eligible_for_v3_ab(blocks, is_script_run=False) is False
+ assert run_is_eligible_for_v3_ab(blocks) is False
provider = FakeExperimentationProvider({WORKFLOW_TASK_V3_AB_FLAG: True})
await resolve_arm(
@@ -191,7 +194,7 @@ def test_pinned_non_default_engine_block_disqualifies_the_run(pinned_engine: Run
# in explicitly, not a treatment exposure.
pinned = _make_block(NavigationBlock, label="pinned", navigation_goal="Apply to the job", engine=pinned_engine)
- assert run_is_eligible_for_v3_ab([eligible, pinned], is_script_run=False) is False
+ assert run_is_eligible_for_v3_ab([eligible, pinned]) is False
@pytest.mark.asyncio
@@ -201,7 +204,7 @@ async def test_all_eligible_run_resolves_every_block_to_treatment(scoped_context
_make_block(NavigationBlock, label="t2", navigation_goal="Apply to the job"),
_make_block(ActionBlock, label="t3"),
]
- assert run_is_eligible_for_v3_ab(blocks, is_script_run=False) is True
+ assert run_is_eligible_for_v3_ab(blocks) is True
provider = FakeExperimentationProvider({WORKFLOW_TASK_V3_AB_FLAG: True})
await resolve_arm(scoped_context, provider, workflow_run_id="wr_treatment", ineligibility_reason=None)
@@ -364,14 +367,6 @@ async def test_an_ab_flag_local_evaluation_cannot_answer_is_control_but_not_a_bu
assert resolution.log["route_reason"] == WorkflowBlockEngineRouteReason.flag_undefined
-def test_script_run_is_never_eligible() -> None:
- blocks: list[BaseTaskBlock] = [
- _make_block(TaskBlock, label="t1"),
- _make_block(NavigationBlock, label="t2", navigation_goal="Apply to the job"),
- ]
- assert run_is_eligible_for_v3_ab(blocks, is_script_run=True) is False
-
-
@pytest.mark.asyncio
async def test_noop_provider_never_queried_and_leaves_engine_unchanged(scoped_context: SkyvernContext) -> None:
provider = NoOpExperimentationProvider()
@@ -401,7 +396,7 @@ def test_non_task_blocks_ignored_but_nested_loop_task_blocks_considered() -> Non
eligible = _make_block(TaskBlock, label="ok")
flat_without_loop = get_all_blocks([code_block, eligible])
- assert run_is_eligible_for_v3_ab(flat_without_loop, is_script_run=False) is True
+ assert run_is_eligible_for_v3_ab(flat_without_loop) is True
ineligible_nested = _make_block(TaskBlock, label="nested_pinned", engine=RunEngine.openai_cua)
loop = ForLoopBlock(
@@ -412,7 +407,7 @@ def test_non_task_blocks_ignored_but_nested_loop_task_blocks_considered() -> Non
flat_with_loop = get_all_blocks([code_block, eligible, loop])
assert ineligible_nested in flat_with_loop
- assert run_is_eligible_for_v3_ab(flat_with_loop, is_script_run=False) is False
+ assert run_is_eligible_for_v3_ab(flat_with_loop) is False
def test_inert_blocks_do_not_disqualify_an_otherwise_eligible_run() -> None:
@@ -422,7 +417,7 @@ def test_inert_blocks_do_not_disqualify_an_otherwise_eligible_run() -> None:
human_block = _make_block(HumanInteractionBlock, label="human")
eligible = _make_block(TaskBlock, label="ok")
- assert run_is_eligible_for_v3_ab([url_block, human_block, eligible], is_script_run=False) is True
+ assert run_is_eligible_for_v3_ab([url_block, human_block, eligible]) is True
def test_run_with_only_inert_or_non_task_blocks_is_not_eligible() -> None:
@@ -430,7 +425,7 @@ def test_run_with_only_inert_or_non_task_blocks_is_not_eligible() -> None:
url_block = _make_block(UrlBlock, label="goto", url="https://example.com")
human_block = _make_block(HumanInteractionBlock, label="human")
- assert run_is_eligible_for_v3_ab([code_block, url_block, human_block], is_script_run=False) is False
+ assert run_is_eligible_for_v3_ab([code_block, url_block, human_block]) is False
@pytest.mark.asyncio
@@ -516,6 +511,63 @@ async def test_base_task_block_execute_dispatches_with_the_resolved_engine(scope
assert captured["engine"] == RunEngine.skyvern_v3
+async def test_a_retry_after_a_step_engine_fallback_runs_v3_under_a_v3_row(scoped_context: SkyvernContext) -> None:
+ # Two attempts share one row. Attempt 1 falls back to the step engine (an unevaluable kill switch),
+ # which relabels the row v1 and fails; attempt 2 dispatches the pin on v3 again.
+ block = _make_block(TaskBlock, label="pinned_retry", engine=RunEngine.skyvern_v3, max_retries=1)
+ row = {"engine": RunEngine.skyvern_v3.value}
+ dispatches: list[tuple[RunEngine, str]] = []
+
+ async def _update_row(**kwargs: Any) -> Any:
+ if kwargs.get("engine") is not None:
+ row["engine"] = kwargs["engine"]
+ return MagicMock()
+
+ async def _execute_step(**kwargs: Any) -> Any:
+ dispatches.append((kwargs["engine"], row["engine"]))
+ if len(dispatches) == 1:
+ row["engine"] = RunEngine.skyvern_v1.value
+ return None
+ raise _EngineCaptured()
+
+ now = datetime.now(UTC)
+ failed_task = make_task(now, make_organization(now), status=TaskStatus.failed, failure_reason="step failed")
+ with _mock_block_execute_deps(working_page_url="https://example.com/dashboard") as deps:
+ deps["agent"].execute_step = AsyncMock(side_effect=_execute_step)
+ deps["observer_db"].update_workflow_run_block = AsyncMock(side_effect=_update_row)
+ deps["tasks_db"].get_task = AsyncMock(return_value=failed_task)
+ with (
+ patch(
+ "skyvern.forge.sdk.workflow.models.block.app.STORAGE",
+ new=MagicMock(
+ get_current_attempt_downloaded_files=AsyncMock(return_value=[]),
+ get_downloaded_file_signature_aliases=MagicMock(return_value=()),
+ ),
+ create=True,
+ ),
+ patch(
+ "skyvern.forge.sdk.workflow.models.block.app.WORKFLOW_SERVICE.get_recent_task_screenshot_artifacts",
+ new=AsyncMock(return_value=[]),
+ ),
+ patch(
+ "skyvern.forge.sdk.workflow.models.block.app.WORKFLOW_SERVICE.get_recent_workflow_screenshot_artifacts",
+ new=AsyncMock(return_value=[]),
+ ),
+ pytest.raises(_EngineCaptured),
+ ):
+ await block.execute(
+ workflow_run_id="wr_missing_starter_url_test",
+ workflow_run_block_id="wrb_test",
+ organization_id="o_test",
+ )
+
+ assert dispatches == [
+ (RunEngine.skyvern_v3, RunEngine.skyvern_v3.value),
+ (RunEngine.skyvern_v3, RunEngine.skyvern_v3.value),
+ ]
+
+
async def _persisted_engine_from_execute_workflow_blocks(
monkeypatch: pytest.MonkeyPatch,
provider: BaseExperimentationProvider,
@@ -525,6 +577,8 @@ async def _persisted_engine_from_execute_workflow_blocks(
first_version_created_at: datetime | None = None,
trigger_type: WorkflowRunTriggerType | None = WorkflowRunTriggerType.api,
block: BaseTaskBlock | None = None,
+ script: Any = None,
+ code_gen: bool | None = None,
) -> RunEngine:
"""Drives the real WorkflowService._execute_workflow_blocks -> _execute_single_block ->
Block.execute_safe chain for a single eligible TaskBlock and returns the engine it persisted,
@@ -552,6 +606,7 @@ async def _persisted_engine_from_execute_workflow_blocks(
workflow_run.run_with = None
workflow_run.retried_from_workflow_run_id = None
workflow_run.trigger_type = trigger_type
+ workflow_run.code_gen = code_gen
organization = MagicMock()
organization.organization_id = "org_e2e"
@@ -579,13 +634,14 @@ async def _persisted_engine_from_execute_workflow_blocks(
)
service = WorkflowService()
- monkeypatch.setattr(service, "should_run_script", AsyncMock(return_value=False))
+ monkeypatch.setattr(service, "should_run_script", AsyncMock(return_value=script is not None))
with pytest.raises(_EngineCaptured):
await service._execute_workflow_blocks(
workflow=workflow,
workflow_run=workflow_run,
organization=organization,
+ script=script,
)
return captured["engine"]
@@ -759,22 +815,20 @@ def _jinja_condition_while_loop_blocks() -> list[Block]:
@pytest.mark.parametrize(
- ("blocks_factory", "is_script_run", "expected_reason"),
+ ("blocks_factory", "expected_reason"),
[
- (_eligible_mix_blocks, True, V3AbIneligibleReason.script_run),
- (_pinned_engine_blocks, False, V3AbIneligibleReason.pinned_engine),
- (_unsupported_block_type_blocks, False, V3AbIneligibleReason.unsupported_block),
- (_download_gated_validation_blocks, False, V3AbIneligibleReason.unsupported_block),
- (_totp_blocks, False, None),
- (_no_reroutable_blocks, False, V3AbIneligibleReason.no_reroutable_blocks),
- (_eligible_mix_blocks, False, None),
- (_prompt_branch_conditional_only_blocks, False, None),
- (_jinja_only_conditional_blocks, False, V3AbIneligibleReason.no_reroutable_blocks),
- (_prompt_condition_while_loop_only_blocks, False, None),
- (_jinja_condition_while_loop_blocks, False, V3AbIneligibleReason.no_reroutable_blocks),
+ (_pinned_engine_blocks, V3AbIneligibleReason.pinned_engine),
+ (_unsupported_block_type_blocks, V3AbIneligibleReason.unsupported_block),
+ (_download_gated_validation_blocks, V3AbIneligibleReason.unsupported_block),
+ (_totp_blocks, None),
+ (_no_reroutable_blocks, V3AbIneligibleReason.no_reroutable_blocks),
+ (_eligible_mix_blocks, None),
+ (_prompt_branch_conditional_only_blocks, None),
+ (_jinja_only_conditional_blocks, V3AbIneligibleReason.no_reroutable_blocks),
+ (_prompt_condition_while_loop_only_blocks, None),
+ (_jinja_condition_while_loop_blocks, V3AbIneligibleReason.no_reroutable_blocks),
],
ids=[
- "script_run",
"pinned_engine",
"unsupported_block_type",
"unsupported_block_validation_download",
@@ -789,14 +843,13 @@ def _jinja_condition_while_loop_blocks() -> list[Block]:
)
def test_v3_ab_ineligibility_reason_maps_each_disqualifier(
blocks_factory: Callable[[], list[BaseTaskBlock]],
- is_script_run: bool,
expected_reason: V3AbIneligibleReason | None,
) -> None:
blocks = blocks_factory()
- reason = v3_ab_ineligibility_reason(blocks, is_script_run=is_script_run)
+ reason = v3_ab_ineligibility_reason(blocks)
assert reason == expected_reason
# run_is_eligible_for_v3_ab must stay a thin wrapper over the same decision.
- assert run_is_eligible_for_v3_ab(blocks, is_script_run=is_script_run) is (expected_reason is None)
+ assert run_is_eligible_for_v3_ab(blocks) is (expected_reason is None)
@pytest.mark.asyncio
@@ -1039,7 +1092,7 @@ async def test_a_run_that_never_reaches_the_ab_pays_for_no_tier_lookup(scoped_co
# kill switch stops: a Redis or pooler incident is exactly when that switch gets flipped, and
# this resolver holds a lock while it runs.
provider = FakeExperimentationProvider({WORKFLOW_TASK_V3_AB_FLAG: True})
- for reason, run_id in ((V3AbIneligibleReason.script_run, "wr_inelig"), (None, "wr_killed")):
+ for reason, run_id in ((V3AbIneligibleReason.unsupported_block, "wr_inelig"), (None, "wr_killed")):
provider.flags[DISABLE_TASK_V3_FLAG] = reason is None
tier = AsyncMock(return_value=BillingTier.ENTERPRISE)
with (
@@ -1354,13 +1407,13 @@ async def test_ineligible_run_is_never_defaulted_to_v3(
scoped_context: SkyvernContext, v3_default_cutoff: datetime
) -> None:
# Run-level eligibility is about whether the run can be rerouted at all, so the rule changes the
- # arm decision only: a script run or a pinned block still executes as authored.
+ # arm decision only: an unsupported or pinned block still executes as authored.
provider = FakeExperimentationProvider({WORKFLOW_TASK_V3_AB_FLAG: True})
resolution = await resolve_arm(
scoped_context,
provider,
workflow_run_id="wr_new_but_ineligible",
- ineligibility_reason=V3AbIneligibleReason.script_run,
+ ineligibility_reason=V3AbIneligibleReason.unsupported_block,
billing_tier=BillingTier.SELF_SERVE,
first_version_created_at=v3_default_cutoff.replace(tzinfo=None) + timedelta(days=1),
)
@@ -1435,7 +1488,7 @@ async def _resolve_chosen_engine_run(
context,
provider,
workflow_run_id=run_id,
- ineligibility_reason=v3_ab_ineligibility_reason(blocks, is_script_run=False),
+ ineligibility_reason=v3_ab_ineligibility_reason(blocks),
takes_default_engine=takes_default_engine(blocks),
billing_tier=BillingTier.ENTERPRISE,
first_version_created_at=born_at,
@@ -1577,7 +1630,7 @@ def test_an_omitted_engine_is_unset_and_a_stored_one_is_kept() -> None:
assert block.engine is None
assert type(block).model_validate(block.model_dump()).engine is None
# Unset routes like skyvern_v1 before the cutoff, so it never reads as a pin.
- assert v3_ab_ineligibility_reason([block], is_script_run=False) is None
+ assert v3_ab_ineligibility_reason([block]) is None
assert takes_default_engine([block]) is True
@@ -1712,3 +1765,105 @@ def test_a_pinned_skyvern_v1_survives_the_stored_definition_and_counts_as_a_chan
assert WorkflowDefinition.model_validate(pinned.model_dump(mode="json")).blocks[0].engine_pinned is True
# Before the cutoff an unmarked skyvern-1.0 and an unset engine compare alike; a pin must not.
assert workflow_definitions_differ(unmarked, pinned)
+
+
+ ("flags", "expected_reason", "displaced"),
+ [
+ ({WORKFLOW_TASK_V3_AB_FLAG: True}, WorkflowBlockEngineRouteReason.code_bucket_treatment, True),
+ ({WORKFLOW_TASK_V3_AB_FLAG: False}, WorkflowBlockEngineRouteReason.code_bucket_control, False),
+ ({}, WorkflowBlockEngineRouteReason.flag_undefined, False),
+ ],
+)
+async def test_a_code_mode_run_takes_the_ab_coin_and_only_treatment_leaves_its_script(
+ scoped_context: SkyvernContext,
+ flags: dict[str, bool],
+ expected_reason: WorkflowBlockEngineRouteReason,
+ displaced: bool,
+) -> None:
+ provider = FakeExperimentationProvider(flags)
+
+ resolution = await resolve_arm(
+ scoped_context, provider, workflow_run_id="wr_code", ineligibility_reason=None, would_be_code=True
+ )
+
+ assert resolution.log["route_reason"] == expected_reason
+ assert resolution.log["would_be_code"] is True
+ assert scoped_context.workflow_block_engine_arm_decision.would_be_code is True
+ assert code_mode_displaced("wr_code") is displaced
+ assert code_mode_displaced("wr_other_run") is False
+ # Sent for reads only; no release condition may target it (see _ab_flag_puts_run_in_treatment).
+ [ab_properties] = [props for flag, _, props in provider.calls if flag == WORKFLOW_TASK_V3_AB_FLAG]
+ assert ab_properties[WOULD_BE_CODE_PROPERTY] == "true"
+
+
+async def test_the_new_workflow_defaults_never_take_a_code_mode_run_off_its_script(
+ scoped_context: SkyvernContext, monkeypatch: pytest.MonkeyPatch, v3_default_cutoff: datetime
+) -> None:
+ born_after = v3_default_cutoff.replace(tzinfo=None) + timedelta(days=1)
+ self_serve = await resolve_arm(
+ scoped_context,
+ FakeExperimentationProvider({WORKFLOW_TASK_V3_AB_FLAG: False}),
+ workflow_run_id="wr_self_serve_code",
+ ineligibility_reason=None,
+ billing_tier=BillingTier.SELF_SERVE,
+ first_version_created_at=born_after,
+ would_be_code=True,
+ )
+ assert self_serve.log["route_reason"] == WorkflowBlockEngineRouteReason.code_bucket_control
+
+ monkeypatch.setattr(settings, "TASK_V3_CHOSEN_ENGINE_CUTOFF", v3_default_cutoff)
+ chosen = await resolve_arm(
+ scoped_context,
+ FakeExperimentationProvider({WORKFLOW_TASK_V3_AB_FLAG: True}),
+ workflow_run_id="wr_chosen_code",
+ ineligibility_reason=None,
+ first_version_created_at=born_after,
+ would_be_code=True,
+ )
+ # The cutoff rule keeps routing a script run's uncached blocks and AI fallback, as before.
+ assert chosen.log["route_reason"] == WorkflowBlockEngineRouteReason.new_workflow_v3_default
+ assert code_mode_displaced("wr_self_serve_code") is False
+ assert code_mode_displaced("wr_chosen_code") is False
+
+
+async def test_a_treatment_code_mode_run_executes_on_v3_without_loading_or_minting_a_script(
+ scoped_context: SkyvernContext, monkeypatch: pytest.MonkeyPatch
+) -> None:
+ script = MagicMock(script_id="s_cached", script_revision_id="sr_cached")
+ load_blocks = AsyncMock(return_value=[])
+ monkeypatch.setattr(app.DATABASE.scripts, "get_script_blocks_by_script_revision_id", load_blocks)
+ scoped_context.generate_script = True # what a code_gen run carries into execution
+
+ engine = await _persisted_engine_from_execute_workflow_blocks(
+ monkeypatch, FakeExperimentationProvider({WORKFLOW_TASK_V3_AB_FLAG: True}), script=script
+ )
+
+ assert engine == RunEngine.skyvern_v3
+ load_blocks.assert_not_awaited()
+ assert scoped_context.script_id is None
+ assert scoped_context.generate_script is False
+
+
[email protected]("born_after_chosen_engine_cutoff", [False, True])
+async def test_an_explicit_code_generation_request_stays_on_v1(
+ scoped_context: SkyvernContext, monkeypatch: pytest.MonkeyPatch, born_after_chosen_engine_cutoff: bool
+) -> None:
+ # A run asked to produce a script must not land on v3, whose actions yield no usable one: neither by
+ # the A/B nor by the chosen-engine rule for new workflows.
+ born_at = None
+ if born_after_chosen_engine_cutoff:
+ monkeypatch.setattr(settings, "TASK_V3_CHOSEN_ENGINE_CUTOFF", CHOSEN_ENGINE_CUTOFF)
+ born_at = CHOSEN_ENGINE_CUTOFF.replace(tzinfo=None) + timedelta(days=1)
+ engine = await _persisted_engine_from_execute_workflow_blocks(
+ monkeypatch,
+ FakeExperimentationProvider({WORKFLOW_TASK_V3_AB_FLAG: True}),
+ code_gen=True,
+ first_version_created_at=born_at,
+ )
+
+ assert engine == RunEngine.skyvern_v1
diff --git a/tests/unit/test_workflow_run_minutes_gate.py b/tests/unit/test_workflow_run_minutes_gate.py
index c19db6491..1549aaa60 100644
--- a/tests/unit/test_workflow_run_minutes_gate.py
+++ b/tests/unit/test_workflow_run_minutes_gate.py
@@ -730,6 +730,8 @@ async def test_duration_log_reads_unknown_for_a_different_runs_context(
duration_logs = [e for e in logs if e.get("event") == "Workflow run duration metrics"]
assert len(duration_logs) == 1
assert duration_logs[0]["task_v3_ab_arm"] == "unknown"
+ # Null, not false: a code-mode run whose attribution was lost must not read as an agent-mode run.
+ assert duration_logs[0]["would_be_code"] is None
# And no route_reason, which is why the single-log arm read documented in
# cloud_docs/feature-flags/task-v3-billing-tier-rollout.md cannot see this population at all: a
# route_reason filter drops these rows from both arms, so they have to be counted and
@@ -822,8 +824,18 @@ _V3_DEFAULT_CUTOFF = datetime(2026, 9, 15, tzinfo=UTC)
"billing_tier": None,
},
),
+ (
+ None,
+ {WORKFLOW_TASK_V3_AB_FLAG: True},
+ {"ineligibility_reason": None, "billing_tier": BillingTier.ENTERPRISE, "would_be_code": True},
+ {
+ "task_v3_ab_arm": "treatment",
+ "route_reason": WorkflowBlockEngineRouteReason.code_bucket_treatment,
+ "would_be_code": True,
+ },
+ ),
],
- ids=["bucketed-treatment", "bucketed-control", "new-workflow-default", "ineligible"],
+ ids=["bucketed-treatment", "bucketed-control", "new-workflow-default", "ineligible", "code-mode-treatment"],
)
async def test_duration_log_carries_the_tier_and_route_reason_the_run_was_bucketed_on(
monkeypatch: pytest.MonkeyPatch,