Skip to content

Automate browser based workflows with AI

Honor the chosen block engine for workflows created after a cutoff (backend) (#8732)

pedrohsdba9bed62b14d2
diff --git a/docs/api-reference/openapi.json b/docs/api-reference/openapi.json
index e1c975fd0..e4fc9b113 100644
--- a/docs/api-reference/openapi.json
+++ b/docs/api-reference/openapi.json
@@ -8942,8 +8942,14 @@
"default": ""
},
"engine": {
- "$ref": "#/components/schemas/RunEngine",
- "default": "skyvern-1.0"
+ "anyOf": [
+ {
+ "$ref": "#/components/schemas/RunEngine"
+ },
+ {
+ "type": "null"
+ }
+ ]
},
"complete_criterion": {
"anyOf": [
@@ -9257,8 +9263,14 @@
"default": ""
},
"engine": {
- "$ref": "#/components/schemas/RunEngine",
- "default": "skyvern-1.0"
+ "anyOf": [
+ {
+ "$ref": "#/components/schemas/RunEngine"
+ },
+ {
+ "type": "null"
+ }
+ ]
},
"navigation_goal": {
"anyOf": [
@@ -14455,8 +14467,14 @@
"default": ""
},
"engine": {
- "$ref": "#/components/schemas/RunEngine",
- "default": "skyvern-1.0"
+ "anyOf": [
+ {
+ "$ref": "#/components/schemas/RunEngine"
+ },
+ {
+ "type": "null"
+ }
+ ]
},
"complete_criterion": {
"anyOf": [
@@ -14792,8 +14810,14 @@
"default": ""
},
"engine": {
- "$ref": "#/components/schemas/RunEngine",
- "default": "skyvern-1.0"
+ "anyOf": [
+ {
+ "$ref": "#/components/schemas/RunEngine"
+ },
+ {
+ "type": "null"
+ }
+ ]
},
"data_schema": {
"anyOf": [
@@ -15205,8 +15229,14 @@
"default": ""
},
"engine": {
- "$ref": "#/components/schemas/RunEngine",
- "default": "skyvern-1.0"
+ "anyOf": [
+ {
+ "$ref": "#/components/schemas/RunEngine"
+ },
+ {
+ "type": "null"
+ }
+ ]
},
"complete_criterion": {
"anyOf": [
@@ -15742,8 +15772,14 @@
"default": ""
},
"engine": {
- "$ref": "#/components/schemas/RunEngine",
- "default": "skyvern-1.0"
+ "anyOf": [
+ {
+ "$ref": "#/components/schemas/RunEngine"
+ },
+ {
+ "type": "null"
+ }
+ ]
},
"error_code_mapping": {
"anyOf": [
@@ -18630,7 +18666,14 @@
"default": ""
},
"engine": {
- "$ref": "#/components/schemas/RunEngine",
+ "anyOf": [
+ {
+ "$ref": "#/components/schemas/RunEngine"
+ },
+ {
+ "type": "null"
+ }
+ ],
"default": "skyvern-1.0"
},
"complete_criterion": {
@@ -19443,8 +19486,14 @@
"default": ""
},
"engine": {
- "$ref": "#/components/schemas/RunEngine",
- "default": "skyvern-1.0"
+ "anyOf": [
+ {
+ "$ref": "#/components/schemas/RunEngine"
+ },
+ {
+ "type": "null"
+ }
+ ]
},
"complete_criterion": {
"anyOf": [
@@ -19748,8 +19797,14 @@
"default": ""
},
"engine": {
- "$ref": "#/components/schemas/RunEngine",
- "default": "skyvern-1.0"
+ "anyOf": [
+ {
+ "$ref": "#/components/schemas/RunEngine"
+ },
+ {
+ "type": "null"
+ }
+ ]
},
"navigation_goal": {
"anyOf": [
@@ -20257,8 +20312,14 @@
"default": ""
},
"engine": {
- "$ref": "#/components/schemas/RunEngine",
- "default": "skyvern-1.0"
+ "anyOf": [
+ {
+ "$ref": "#/components/schemas/RunEngine"
+ },
+ {
+ "type": "null"
+ }
+ ]
},
"complete_criterion": {
"anyOf": [
@@ -20555,8 +20616,14 @@
"default": ""
},
"engine": {
- "$ref": "#/components/schemas/RunEngine",
- "default": "skyvern-1.0"
+ "anyOf": [
+ {
+ "$ref": "#/components/schemas/RunEngine"
+ },
+ {
+ "type": "null"
+ }
+ ]
},
"error_code_mapping": {
"anyOf": [
@@ -24442,8 +24509,14 @@
"default": ""
},
"engine": {
- "$ref": "#/components/schemas/RunEngine",
- "default": "skyvern-1.0"
+ "anyOf": [
+ {
+ "$ref": "#/components/schemas/RunEngine"
+ },
+ {
+ "type": "null"
+ }
+ ]
},
"complete_criterion": {
"anyOf": [
@@ -24742,8 +24815,14 @@
"default": ""
},
"engine": {
- "$ref": "#/components/schemas/RunEngine",
- "default": "skyvern-1.0"
+ "anyOf": [
+ {
+ "$ref": "#/components/schemas/RunEngine"
+ },
+ {
+ "type": "null"
+ }
+ ]
},
"navigation_goal": {
"anyOf": [
@@ -27396,7 +27475,14 @@
"default": ""
},
"engine": {
- "$ref": "#/components/schemas/RunEngine",
+ "anyOf": [
+ {
+ "$ref": "#/components/schemas/RunEngine"
+ },
+ {
+ "type": "null"
+ }
+ ],
"default": "skyvern-1.0"
},
"complete_criterion": {
@@ -27839,8 +27925,14 @@
"default": ""
},
"engine": {
- "$ref": "#/components/schemas/RunEngine",
- "default": "skyvern-1.0"
+ "anyOf": [
+ {
+ "$ref": "#/components/schemas/RunEngine"
+ },
+ {
+ "type": "null"
+ }
+ ]
},
"complete_criterion": {
"anyOf": [
@@ -28128,8 +28220,14 @@
"default": false
},
"engine": {
- "$ref": "#/components/schemas/RunEngine",
- "default": "skyvern-1.0"
+ "anyOf": [
+ {
+ "$ref": "#/components/schemas/RunEngine"
+ },
+ {
+ "type": "null"
+ }
+ ]
},
"complete_criterion": {
"anyOf": [
@@ -29561,6 +29659,17 @@
"title": "Copilot Authored",
"default": false
},
+ "effective_default_engine": {
+ "anyOf": [
+ {
+ "$ref": "#/components/schemas/RunEngine"
+ },
+ {
+ "type": "null"
+ }
+ ],
+ "description": "The engine a task block with no `engine` set runs on in this agent, or null when engine routing decides it. Populated by the detail endpoint only."
+ },
"original_created_by": {
"anyOf": [
{
diff --git a/skyvern/config.py b/skyvern/config.py
index bcaba82f0..fd49b4f31 100644
--- a/skyvern/config.py
+++ b/skyvern/config.py
@@ -591,6 +591,11 @@ class Settings(BaseSettings):
# which covers the job recipe endpoints because those build a published definition. Enrolled runs
# are not randomized, so every per-arm read must exclude them by route_reason.
TASK_V3_DEFAULT_ENGINE_WORKFLOW_CUTOFF: datetime | None = None
+ # Workflows whose permanent id was born at or after this instant, in every billing tier, honor each
+ # task block's chosen engine and run a block with no engine on Task V3, outside WORKFLOW_TASK_V3_AB.
+ # Excludes the same per-call platform workflows as the cutoff above. None disables the rule, as does
+ # having no flag provider; DISABLE_TASK_V3 still wins. A naive value is read as UTC.
+ TASK_V3_CHOSEN_ENGINE_CUTOFF: datetime | None = None
# VOLCENGINE (Doubao)
ENABLE_VOLCENGINE: bool = False
@@ -1078,7 +1083,7 @@ class Settings(BaseSettings):
return None
return value
- @field_validator("TASK_V3_DEFAULT_ENGINE_WORKFLOW_CUTOFF", mode="before")
+ @field_validator("TASK_V3_DEFAULT_ENGINE_WORKFLOW_CUTOFF", "TASK_V3_CHOSEN_ENGINE_CUTOFF", mode="before")
@classmethod
def _task_v3_default_engine_workflow_cutoff_off_sentinels(cls, value: Any) -> Any:
# This setting is the rule's settings-side kill path, and blanking an already-set env var is
diff --git a/skyvern/forge/sdk/experimentation/workflow_block_engine.py b/skyvern/forge/sdk/experimentation/workflow_block_engine.py
index 4fdca7ee5..572695789 100644
--- a/skyvern/forge/sdk/experimentation/workflow_block_engine.py
+++ b/skyvern/forge/sdk/experimentation/workflow_block_engine.py
@@ -46,6 +46,11 @@ class WorkflowBlockEngineRouteReason(StrEnum):
flag_bucket_treatment = "flag_bucket_treatment"
flag_bucket_control = "flag_bucket_control"
new_self_serve_workflow_default = "new_self_serve_workflow_default"
+ # A workflow born at or after TASK_V3_CHOSEN_ENGINE_CUTOFF, never randomized: at least one block
+ # left its engine unset and runs on v3 (``new_workflow_v3_default``), or every block's engine was
+ # chosen and is honored as written (``chosen_engine``).
+ new_workflow_v3_default = "new_workflow_v3_default"
+ chosen_engine = "chosen_engine"
ineligible = "ineligible"
disabled = "disabled"
flag_undefined = "flag_undefined"
@@ -80,6 +85,8 @@ class WorkflowBlockEngineArmDecision:
route_reason: WorkflowBlockEngineRouteReason | None = None
billing_tier: BillingTier | None = None
+ # An explicit skyvern_v1 on a block is a pin, not a routable default (see Block.resolve_engine).
+ honors_chosen_engine: bool = False
NO_ARM_DECISION = WorkflowBlockEngineArmDecision()
@@ -144,15 +151,14 @@ def _as_utc(value: datetime) -> datetime:
return value.replace(tzinfo=UTC) if value.tzinfo is None else value.astimezone(UTC)
-async def _workflow_is_new_for_v3_default(workflow_permanent_id: str | None, organization_id: str | None) -> bool:
- """Whether this permanent id was born at or after the v3-default cutoff.
+async def _workflow_birth(workflow_permanent_id: str | None, organization_id: str | None) -> datetime | None:
+ """When this permanent id was born, in UTC, or None when that could not be read.
- Never raises and never answers True on a read it could not complete: the rule can only add
- treatment, so any doubt has to leave the run on the arm the A/B would have given it.
+ Never raises, and None satisfies no cutoff: both cutoff rules take a run out of the A/B, so any
+ doubt has to leave the run on the arm the A/B would have given it.
"""
- cutoff = settings.TASK_V3_DEFAULT_ENGINE_WORKFLOW_CUTOFF
- if cutoff is None or not workflow_permanent_id or not organization_id:
- return False
+ if not workflow_permanent_id or not organization_id:
+ return None
try:
# The earliest version's timestamp, deleted versions included, is when the permanent id was
# born. Reading the version this run executes would instead enrol every long-lived workflow
@@ -160,18 +166,30 @@ async def _workflow_is_new_for_v3_default(workflow_permanent_id: str | None, org
born_at = await app.DATABASE.workflows.get_workflow_permanent_id_created_at(
workflow_permanent_id, organization_id
)
- if born_at is None:
- return False
- # Inside the try with the read: a rule that raised here would reach the resolver's catch-all
- # and send the run to control, which is a different arm than the A/B would have given it.
- return _as_utc(born_at) >= _as_utc(cutoff)
+ # Inside the try with the read: a conversion that raised here would reach the resolver's
+ # catch-all and send the run to control, a different arm than the A/B would have given it.
+ return None if born_at is None else _as_utc(born_at)
except Exception:
LOG.warning(
- "Failed to read the workflow's birth timestamp for the Task V3 default rule",
+ "Failed to read the workflow's birth timestamp for a Task V3 cutoff rule",
workflow_permanent_id=workflow_permanent_id,
exc_info=True,
)
- return False
+ return None
+
+
+def _born_at_or_after(born_at: datetime | None, cutoff: datetime | None) -> bool:
+ return born_at is not None and cutoff is not None and born_at >= _as_utc(cutoff)
+
+
+def _is_kept_workflow_run(workflow_status: WorkflowStatus, trigger_type: WorkflowRunTriggerType | None) -> bool:
+ # An auto_generated running version is a per-call workflow the login, download_files, credential
+ # test-login and SDK endpoints mint and run in one request, and enrolling those would move standing
+ # API traffic rather than the workflows a customer keeps; the birth version's status would be the
+ # wrong test, because the prompt box writes version 1 as auto_generated and the editor save that
+ # keeps it writes the next as published. The job-recipe endpoints are per-call too but build a
+ # published definition, so only their trigger kind excludes them.
+ return workflow_status != WorkflowStatus.auto_generated and trigger_type not in PER_CALL_WORKFLOW_RUN_TRIGGER_TYPES
async def _ab_flag_puts_run_in_treatment(
@@ -213,6 +231,7 @@ async def resolve_workflow_block_engine_arm(
workflow_status: WorkflowStatus,
trigger_type: WorkflowRunTriggerType | None,
ineligibility_reason: V3AbIneligibleReason | None,
+ takes_default_engine: bool | None,
) -> None:
"""Resolve the workflow-block engine A/B once at execution start and pin the arm on the context.
@@ -228,6 +247,13 @@ async def resolve_workflow_block_engine_arm(
consulted, so those runs are marked ``new_self_serve_workflow_default`` for every per-arm read to
exclude. No flag gates the rule: ``DISABLE_TASK_V3`` or unsetting the cutoff turns it off.
+ A workflow of any tier born at or after ``settings.TASK_V3_CHOSEN_ENGINE_CUTOFF`` skips the A/B
+ and the rule above: every block's chosen engine is honored and an unset one runs on v3.
+ ``takes_default_engine`` (see ``block.takes_default_engine``) says whether any block left its
+ engine unset, or is None when the run has no engine to choose. The rule does not require A/B
+ eligibility, since a pinned block no longer keeps the run's unset blocks on v1, and so covers a
+ script run's uncached blocks and AI fallback.
+
``workflow_status`` is the status of the version this run executes and ``trigger_type`` is how
the run was launched. Together they are what separates a workflow a customer keeps from the
per-call throwaways several endpoints mint and run in one request: most of those run while still
@@ -247,11 +273,27 @@ async def resolve_workflow_block_engine_arm(
if context.workflow_block_engine_resolved_run_id == workflow_run_id:
return
override: RunEngine | None = None
+ honors_chosen_engine = False
run_is_eligible = ineligibility_reason is None
billing_tier: BillingTier | None = None
route_reason = WorkflowBlockEngineRouteReason.ineligible
+ chosen_engine_candidate = (
+ settings.TASK_V3_CHOSEN_ENGINE_CUTOFF is not None
+ and takes_default_engine is not None
+ and _is_kept_workflow_run(workflow_status, trigger_type)
+ )
+ # Both cutoff rules compare the same birth timestamp, so it is read at most once per run.
+ birth_reads: list[datetime | None] = []
+
+ async def born_at_or_after(cutoff: datetime | None) -> bool:
+ if cutoff is None:
+ return False
+ if not birth_reads:
+ birth_reads.append(await _workflow_birth(workflow_permanent_id, organization_id))
+ return _born_at_or_after(birth_reads[0], cutoff)
+
try:
- if run_is_eligible:
+ if run_is_eligible or chosen_engine_candidate:
# The kill switch, shared with the dispatch gate via task_v3_disabled so both
# evaluations use the same cache key, wins over the experiment and over the
# new-workflow default below. A kill flipped mid-run still takes effect at dispatch
@@ -260,27 +302,27 @@ async def resolve_workflow_block_engine_arm(
# when someone flips this, and this lock is held throughout, so the kill switch must
# not queue behind that call.
if await task_v3_disabled(workflow_run_id, organization_id):
- route_reason = WorkflowBlockEngineRouteReason.disabled
- else:
+ if run_is_eligible:
+ route_reason = WorkflowBlockEngineRouteReason.disabled
+ elif chosen_engine_candidate and await born_at_or_after(settings.TASK_V3_CHOSEN_ENGINE_CUTOFF):
+ honors_chosen_engine = True
+ if takes_default_engine:
+ override = RunEngine.skyvern_v3
+ route_reason = WorkflowBlockEngineRouteReason.new_workflow_v3_default
+ else:
+ route_reason = WorkflowBlockEngineRouteReason.chosen_engine
+ elif run_is_eligible:
# billing_tier rides along so a release condition can hold enterprise and
# self-serve at different percentages. Deliberately not passed to
# task_v3_disabled: that flag's two callers must build identical provider cache
# keys, see its docstring.
billing_tier = await _billing_tier_for_arm(organization_id)
# The tier, the status and the trigger are checked first, so anything that
- # cannot use the rule pays for no workflow read. An auto_generated running
- # version is a per-call workflow the login, download_files, credential
- # test-login and SDK endpoints mint and run in one request, and enrolling those
- # would move standing API traffic rather than the workflows a customer keeps;
- # the birth version's status would be the wrong test, because the prompt box
- # writes version 1 as auto_generated and the editor save that keeps it writes
- # the next as published. The job-recipe endpoints are per-call too but build a
- # published definition, so only their trigger kind excludes them.
+ # cannot use the rule pays for no workflow read.
takes_new_workflow_default = (
billing_tier == BillingTier.SELF_SERVE
- and workflow_status != WorkflowStatus.auto_generated
- and trigger_type not in PER_CALL_WORKFLOW_RUN_TRIGGER_TYPES
- and await _workflow_is_new_for_v3_default(workflow_permanent_id, organization_id)
+ and _is_kept_workflow_run(workflow_status, trigger_type)
+ and await born_at_or_after(settings.TASK_V3_DEFAULT_ENGINE_WORKFLOW_CUTOFF)
)
if takes_new_workflow_default:
# A workflow born at or after the cutoff runs its task blocks on v3 by
@@ -312,10 +354,12 @@ async def resolve_workflow_block_engine_arm(
exc_info=True,
)
override = None
+ honors_chosen_engine = False
route_reason = WorkflowBlockEngineRouteReason.flag_error
decision = WorkflowBlockEngineArmDecision(
route_reason=route_reason,
billing_tier=billing_tier,
+ honors_chosen_engine=honors_chosen_engine,
)
context.workflow_block_engine_override = override
context.workflow_block_engine_arm_decision = decision
@@ -351,6 +395,30 @@ def workflow_block_engine_override(workflow_run_id: str | None) -> RunEngine | N
return context.workflow_block_engine_override
+def run_honors_chosen_engine(workflow_run_id: str | None) -> bool:
+ if not workflow_run_id:
+ return False
+ context = skyvern_context.current()
+ if context is None or context.workflow_block_engine_resolved_run_id != workflow_run_id:
+ return False
+ decision = context.workflow_block_engine_arm_decision
+ return decision is not None and decision.honors_chosen_engine
+
+
+async def effective_default_engine(workflow_permanent_id: str, organization_id: str) -> RunEngine | None:
+ """The engine a block with no chosen engine runs on in this workflow, or None when routing decides it.
+
+ Mirrors the resolver's chosen-engine rule for the editor; the per-call exclusions do not apply to
+ a workflow someone opens there.
+ """
+ cutoff = settings.TASK_V3_CHOSEN_ENGINE_CUTOFF
+ if cutoff is None or isinstance(app.EXPERIMENTATION_PROVIDER, NoOpExperimentationProvider):
+ return None
+ if _born_at_or_after(await _workflow_birth(workflow_permanent_id, organization_id), cutoff):
+ return RunEngine.skyvern_v3
+ return None
+
+
def resolved_workflow_block_engine_arm_attribution(workflow_run_id: str | None) -> WorkflowBlockEngineArmAttribution:
"""The arm and the routing facts finalize-time telemetry can attribute to this run.
diff --git a/skyvern/forge/sdk/routes/agent_protocol.py b/skyvern/forge/sdk/routes/agent_protocol.py
index 891bdaadb..311d3fd7e 100644
--- a/skyvern/forge/sdk/routes/agent_protocol.py
+++ b/skyvern/forge/sdk/routes/agent_protocol.py
@@ -78,6 +78,7 @@ from skyvern.forge.sdk.db.repositories.tags import (
from skyvern.forge.sdk.db.repositories.workflows import WorkflowCreationLockTimeout
from skyvern.forge.sdk.enterprise_features import collect_enterprise_gated_run_features
from skyvern.forge.sdk.executor.factory import AsyncExecutorFactory
+from skyvern.forge.sdk.experimentation.workflow_block_engine import effective_default_engine
from skyvern.forge.sdk.models import Step
from skyvern.forge.sdk.routes.code_samples import (
BULK_CANCEL_RUNS_CODE_SAMPLE_PYTHON,
@@ -5382,6 +5383,9 @@ async def get_workflow(
workflow_permanent_id=workflow_permanent_id,
organization_id=current_org.organization_id,
)
+ workflow.effective_default_engine = await effective_default_engine(
+ workflow_permanent_id, current_org.organization_id
+ )
return workflow
diff --git a/skyvern/forge/sdk/workflow/models/block.py b/skyvern/forge/sdk/workflow/models/block.py
index 3e2f0c391..6a708acfe 100644
--- a/skyvern/forge/sdk/workflow/models/block.py
+++ b/skyvern/forge/sdk/workflow/models/block.py
@@ -179,7 +179,10 @@ from skyvern.forge.sdk.db.exceptions import NotFoundError
from skyvern.forge.sdk.db.id import generate_action_id
from skyvern.forge.sdk.experimentation.code_block_ai_fallback import code_block_ai_fallback_flag_enabled
from skyvern.forge.sdk.experimentation.llm_prompt_config import get_llm_handler_for_prompt_type
-from skyvern.forge.sdk.experimentation.workflow_block_engine import workflow_block_engine_override
+from skyvern.forge.sdk.experimentation.workflow_block_engine import (
+ run_honors_chosen_engine,
+ workflow_block_engine_override,
+)
from skyvern.forge.sdk.forge_log import exception_log_fields
from skyvern.forge.sdk.models import Step, StepStatus
from skyvern.forge.sdk.schemas.files import FileInfo
@@ -1879,11 +1882,17 @@ def _should_skip_retry_on_anti_bot_detection(task: Task) -> bool:
return False
+def _engine_is_unset(engine: RunEngine | None) -> bool:
+ return engine is None
+
+
class BaseTaskBlock(Block):
task_type: str = TaskType.general
url: str | None = None
title: str = ""
- engine: RunEngine = RunEngine.skyvern_v1
+ # Left out of the dump when unset, so a stored definition stays readable by an image whose engine
+ # field still rejects null (a rollout or a revert).
+ engine: RunEngine | None = Field(default=None, exclude_if=_engine_is_unset)
complete_criterion: str | None = None
complete_criterion_is_untrusted: bool = False
terminate_criterion: str | None = None
@@ -1974,19 +1983,25 @@ class BaseTaskBlock(Block):
Both the persisted workflow_run_blocks.engine and the execute_step dispatch read this, so
the recorded engine cannot disagree with the one that ran. A block pinned to a non-default
engine is honored as-authored, and a block the eligibility check never saw is left alone;
- neither is ever rerouted.
+ neither is ever rerouted. An unset engine routes like skyvern_v1, except in a run that honors the
+ chosen engine, where an explicit skyvern_v1 is a pin too.
"""
+ declared = self.engine or RunEngine.skyvern_v1
if (
- self.engine != RunEngine.skyvern_v1
- or self._exclude_from_engine_ab
+ self._exclude_from_engine_ab
# Mirrors run_is_eligible_for_v3_ab: a block eligibility skipped as engine-inert must
- # not be labeled v3 here either, or its row claims an engine that never ran. It does not
- # re-check _task_block_supports_v3 because run-level eligibility already rejected the
- # whole run if any block failed it; loosening that predicate means revisiting this.
+ # not be labeled v3 here either, or its row claims an engine that never ran.
or self.block_type in _ENGINE_INERT_BLOCK_TYPES
+ # Redundant for an A/B run, whose eligibility already rejected any such block; a
+ # chosen-engine run skips that check, so a block v3 cannot execute stays off the override.
+ or not _task_block_supports_v3(self)
+ ):
+ return declared
+ if self.engine is not None and (
+ self.engine != RunEngine.skyvern_v1 or run_honors_chosen_engine(workflow_run_id)
):
return self.engine
- return workflow_block_engine_override(workflow_run_id) or self.engine
+ return workflow_block_engine_override(workflow_run_id) or declared
def get_all_parameters(
self,
@@ -14427,6 +14442,9 @@ class HumanInteractionBlock(BaseTaskBlock):
# There is a mypy bug with Literal. Without the type: ignore, mypy will raise an error:
# Parameter 1 of Literal[...] cannot be of type "Any"
block_type: Literal[BlockType.HUMAN_INTERACTION] = BlockType.HUMAN_INTERACTION # type: ignore
+ # Engine-inert, and its YAML has no engine: keeping the stored value makes an unedited re-save
+ # write the definition it read, so the cached script is not cleared.
+ engine: RunEngine | None = Field(default=RunEngine.skyvern_v1, exclude_if=_engine_is_unset)
instructions: str = "Please review and approve or reject to continue the workflow."
positive_descriptor: str = "Approve"
@@ -15390,6 +15408,8 @@ class UrlBlock(BaseTaskBlock):
# Parameter 1 of Literal[...] cannot be of type "Any"
block_type: Literal[BlockType.GOTO_URL] = BlockType.GOTO_URL # type: ignore
url: str
+ # Engine-inert; see HumanInteractionBlock.engine.
+ engine: RunEngine | None = Field(default=RunEngine.skyvern_v1, exclude_if=_engine_is_unset)
class TaskV2Block(Block):
@@ -18946,7 +18966,7 @@ def v3_ab_ineligibility_reason(blocks: list[BlockTypeVar], *, is_script_run: boo
continue
if block.block_type in _ENGINE_INERT_BLOCK_TYPES:
continue
- if block.engine != RunEngine.skyvern_v1:
+ if block.engine not in (None, RunEngine.skyvern_v1):
return V3AbIneligibleReason.pinned_engine
if not _task_block_supports_v3(block):
return V3AbIneligibleReason.unsupported_block
@@ -18958,6 +18978,31 @@ def v3_ab_ineligibility_reason(blocks: list[BlockTypeVar], *, is_script_run: boo
return None
+def takes_default_engine(blocks: list[BlockTypeVar]) -> bool | None:
+ """Whether any block of the run leaves its engine unchosen; None when the run has no engine to choose.
+
+ ``blocks`` is the flattened definition, as for v3_ab_ineligibility_reason. A script run is answered
+ too: its uncached blocks and its AI fallback run on the engine this decides. A prompt-criteria
+ condition counts as unchosen: its synthetic extraction block has no engine an author can set.
+ """
+ has_engine_block = False
+ for block in blocks:
+ if isinstance(block, ConditionalBlock):
+ if any(isinstance(branch.criteria, PromptBranchCriteria) for branch in block.branch_conditions):
+ return True
+ continue
+ if isinstance(block, WhileLoopBlock):
+ if isinstance(block.condition, PromptBranchCriteria):
+ return True
+ continue
+ if not isinstance(block, BaseTaskBlock) or block.block_type in _ENGINE_INERT_BLOCK_TYPES:
+ continue
+ if block.engine is None and _task_block_supports_v3(block):
+ return True
+ has_engine_block = True
+ return False if has_engine_block else None
+
+
def run_is_eligible_for_v3_ab(blocks: list[BlockTypeVar], *, is_script_run: bool) -> bool:
"""Whether a whole workflow run may be rerouted onto v3 by the A/B; see v3_ab_ineligibility_reason."""
return v3_ab_ineligibility_reason(blocks, is_script_run=is_script_run) is None
diff --git a/skyvern/forge/sdk/workflow/models/workflow.py b/skyvern/forge/sdk/workflow/models/workflow.py
index dd8d07137..68852ad46 100644
--- a/skyvern/forge/sdk/workflow/models/workflow.py
+++ b/skyvern/forge/sdk/workflow/models/workflow.py
@@ -32,6 +32,7 @@ from skyvern.forge.sdk.workflow.models.validators import (
normalize_run_metadata,
normalize_run_with,
)
+from skyvern.schemas.run_enums import RunEngine
from skyvern.schemas.runs import (
BROWSER_ADDRESS_SERVER_ASSIGNED_CONTEXT_KEY,
BROWSER_TYPE_ATTACH_CONFLICT_MESSAGE,
@@ -260,6 +261,11 @@ class Workflow(BaseModel):
# Lineage-derived (any version copilot-stamped); populated by the detail GET route only —
# user saves re-stamp created_by/edited_by, so the current version alone is not durable.
copilot_authored: bool = False
+ effective_default_engine: RunEngine | None = Field(
+ default=None,
+ description="The engine a task block with no `engine` set runs on in this agent, or null when "
+ "engine routing decides it. Populated by the detail endpoint only.",
+ )
original_created_by: str | None = Field(
default=None,
description="Who created the agent's first version. Populated by the list endpoint only.",
diff --git a/skyvern/forge/sdk/workflow/service.py b/skyvern/forge/sdk/workflow/service.py
index e098fafcb..fcb01aedb 100644
--- a/skyvern/forge/sdk/workflow/service.py
+++ b/skyvern/forge/sdk/workflow/service.py
@@ -120,6 +120,7 @@ from skyvern.forge.sdk.experimentation.transient_ui_capture import resolve_trans
from skyvern.forge.sdk.experimentation.workflow_block_engine import (
ARM_ATTRIBUTION_LOST,
WorkflowBlockEngineArmAttribution,
+ effective_default_engine,
engine_arm_log_value,
resolve_workflow_block_engine_arm,
resolved_workflow_block_engine_arm_attribution,
@@ -167,6 +168,7 @@ from skyvern.forge.sdk.workflow.exceptions import (
WorkflowVersionConflict,
)
from skyvern.forge.sdk.workflow.models.block import (
+ _ENGINE_INERT_BLOCK_TYPES,
BaseTaskBlock,
Block,
BlockTypeVar,
@@ -188,6 +190,7 @@ from skyvern.forge.sdk.workflow.models.block import (
compute_conditional_scopes,
get_all_blocks,
resolve_conditional_merge_edges,
+ takes_default_engine,
v3_ab_ineligibility_reason,
)
from skyvern.forge.sdk.workflow.models.parameter import (
@@ -1423,7 +1426,9 @@ class ReusedSessionBelowLifetimeFloor(Exception):
self.shortfall = shortfall
-def _get_workflow_definition_core_data(workflow_definition: WorkflowDefinition) -> dict[str, Any]:
+def _get_workflow_definition_core_data(
+ workflow_definition: WorkflowDefinition, *, unset_engine_is_v1: bool = True
+) -> dict[str, Any]:
"""
This function dumps the workflow definition and removes the irrelevant data to the definition, like created_at and modified_at fields inside:
- list of blocks
@@ -1472,6 +1477,15 @@ def _get_workflow_definition_core_data(workflow_definition: WorkflowDefinition)
if current_obj.get("block_type") == BlockType.CODE.value:
for field in code_block_annotation_fields:
current_obj.pop(field, None)
+ # A definition stored before the engine became optional holds skyvern-1.0 where a re-save
+ # of the same blocks leaves it unset. The two route alike on an engine-inert block and
+ # before the chosen-engine cutoff; past it, unset means v3 and skyvern-1.0 is a pin.
+ if (
+ "block_type" in current_obj
+ and current_obj.get("engine") in (None, RunEngine.skyvern_v1.value)
+ and (unset_engine_is_v1 or current_obj["block_type"] in _ENGINE_INERT_BLOCK_TYPES)
+ ):
+ current_obj.pop("engine", None)
# Add all nested dictionaries and lists to queue for processing
for value in current_obj.values():
@@ -7818,6 +7832,7 @@ class WorkflowService:
workflow_status=workflow.status,
trigger_type=workflow_run.trigger_type,
ineligibility_reason=v3_ab_ineligibility_reason(all_blocks, is_script_run=is_script_run),
+ takes_default_engine=takes_default_engine(all_blocks),
)
else:
LOG.warning(
@@ -11142,8 +11157,13 @@ class WorkflowService:
current_definition: dict[str, Any] = {}
new_definition: dict[str, Any] = {}
if previous_valid_workflow:
- current_definition = _get_workflow_definition_core_data(previous_valid_workflow.workflow_definition)
- new_definition = _get_workflow_definition_core_data(workflow_definition)
+ unset_engine_is_v1 = await effective_default_engine(workflow.workflow_permanent_id, organization_id) is None
+ current_definition = _get_workflow_definition_core_data(
+ previous_valid_workflow.workflow_definition, unset_engine_is_v1=unset_engine_is_v1
+ )
+ new_definition = _get_workflow_definition_core_data(
+ workflow_definition, unset_engine_is_v1=unset_engine_is_v1
+ )
has_changes = current_definition != new_definition
# Log definition changes for debugging cache invalidation issues
diff --git a/skyvern/schemas/workflows.py b/skyvern/schemas/workflows.py
index 3be75297d..898123dd7 100644
--- a/skyvern/schemas/workflows.py
+++ b/skyvern/schemas/workflows.py
@@ -840,7 +840,7 @@ class TaskBlockYAML(BlockYAML):
url: str | None = None
title: str = ""
- engine: RunEngine = RunEngine.skyvern_v1
+ engine: RunEngine | None = None
navigation_goal: str | None = None
data_extraction_goal: str | None = None
data_schema: dict[str, Any] | list | str | None = None
@@ -1245,7 +1245,7 @@ class PDFParserBlockYAML(BlockYAML):
class ValidationBlockYAML(BlockYAML):
block_type: Literal[BlockType.VALIDATION] = BlockType.VALIDATION # type: ignore
- engine: RunEngine = RunEngine.skyvern_v1
+ engine: RunEngine | None = None
complete_criterion: str | None = None
terminate_criterion: str | None = None
error_code_mapping: dict[str, str] | None = None
@@ -1260,7 +1260,7 @@ class ActionBlockYAML(BlockYAML):
url: str | None = None
title: str = ""
- engine: RunEngine = RunEngine.skyvern_v1
+ engine: RunEngine | None = None
navigation_goal: str | None = None
selector: str | None = None
ai_fallback: AIFallbackMode = AIFallbackMode.FALLBACK
@@ -1283,7 +1283,7 @@ class NavigationBlockYAML(BlockYAML):
navigation_goal: str
url: str | None = None
title: str = ""
- engine: RunEngine = RunEngine.skyvern_v1
+ engine: RunEngine | None = None
error_code_mapping: dict[str, str] | None = None
max_retries: int = 0
max_steps_per_run: int | None = None
@@ -1308,7 +1308,7 @@ class ExtractionBlockYAML(BlockYAML):
data_extraction_goal: str
url: str | None = None
title: str = ""
- engine: RunEngine = RunEngine.skyvern_v1
+ engine: RunEngine | None = None
data_schema: dict[str, Any] | list | str | None = None
max_retries: int = 0
max_steps_per_run: int | None = None
@@ -1328,7 +1328,7 @@ class LoginBlockYAML(BlockYAML):
url: str | None = None
title: str = ""
- engine: RunEngine = RunEngine.skyvern_v1
+ engine: RunEngine | None = None
navigation_goal: str | None = None
error_code_mapping: dict[str, str] | None = None
max_retries: int = 0
@@ -1409,7 +1409,7 @@ class FileDownloadBlockYAML(BlockYAML):
navigation_goal: str
url: str | None = None
title: str = ""
- engine: RunEngine = RunEngine.skyvern_v1
+ engine: RunEngine | None = None
error_code_mapping: dict[str, str] | None = None
max_retries: int = 0
max_steps_per_run: int | None = None
diff --git a/skyvern/services/pdf_import_service.py b/skyvern/services/pdf_import_service.py
index 7077bab77..9c03fe463 100644
--- a/skyvern/services/pdf_import_service.py
+++ b/skyvern/services/pdf_import_service.py
@@ -144,10 +144,6 @@ class PDFImportService:
if keys_to_include:
blk["parameter_keys"] = keys_to_include
- # Ensure engine where needed
- if blk.get("block_type") in {"navigation", "action", "extraction", "login", "file_download"}:
- blk.setdefault("engine", "skyvern-1.0")
-
# Ensure url exists (can be empty string)
if blk.get("block_type") in {"navigation", "action", "extraction", "file_download"}:
if blk.get("url") is None:
diff --git a/skyvern/services/script_service.py b/skyvern/services/script_service.py
index 53e0a61e5..caf032064 100644
--- a/skyvern/services/script_service.py
+++ b/skyvern/services/script_service.py
@@ -1419,12 +1419,13 @@ async def _detect_user_defined_errors(
return []
-def _resolve_original_block_engine(cache_key: str, workflow: Workflow) -> RunEngine | None:
+def _resolve_original_block_engine(cache_key: str, workflow: Workflow, workflow_run_id: str | None) -> RunEngine | None:
# Recursive: a cached block inside a for/while loop must keep its engine too (labels are
- # validated globally unique, so the first match is the block).
+ # validated globally unique, so the first match is the block). Resolved rather than read, so an
+ # unset engine follows the run's routing like the block itself would.
for block in get_all_blocks(workflow.workflow_definition.blocks):
if block.label == cache_key:
- return block.engine if isinstance(block, BaseTaskBlock) else None
+ return block.resolve_engine(workflow_run_id) if isinstance(block, BaseTaskBlock) else None
return None
@@ -1672,7 +1673,7 @@ async def _fallback_to_ai_run(
# Inherit the original block's engine when the caller left it at default; fail open to v1 on any miss.
if engine == RunEngine.skyvern_v1:
try:
- resolved_engine = _resolve_original_block_engine(cache_key, workflow)
+ resolved_engine = _resolve_original_block_engine(cache_key, workflow, workflow_run_id)
if resolved_engine is not None and resolved_engine != RunEngine.skyvern_v1:
engine = resolved_engine
LOG.debug(
diff --git a/tests/unit/_workflow_block_engine_fakes.py b/tests/unit/_workflow_block_engine_fakes.py
index 6db4b8447..1e3b33588 100644
--- a/tests/unit/_workflow_block_engine_fakes.py
+++ b/tests/unit/_workflow_block_engine_fakes.py
@@ -118,6 +118,7 @@ async def resolve_arm(
first_version_error: Exception | None = None,
workflow_status: WorkflowStatus = WorkflowStatus.published,
trigger_type: WorkflowRunTriggerType | None = WorkflowRunTriggerType.api,
+ takes_default_engine: bool | None = True,
) -> Resolution:
async def read_birth_timestamp(workflow_permanent_id: str, organization_id: str) -> datetime | None:
if first_version_error is not None:
@@ -161,6 +162,7 @@ async def resolve_arm(
workflow_status=workflow_status,
trigger_type=trigger_type,
ineligibility_reason=ineligibility_reason,
+ takes_default_engine=takes_default_engine,
)
logged = dict(mock_log.info.call_args.kwargs) if mock_log.info.call_args else {}
return Resolution(log=logged, birth_reads=birth_reads, warnings=list(mock_log.warning.call_args_list))
diff --git a/tests/unit/test_config.py b/tests/unit/test_config.py
index ad8b5a94e..3bd993356 100644
--- a/tests/unit/test_config.py
+++ b/tests/unit/test_config.py
@@ -133,9 +133,10 @@ def test_api_limit_concurrency_rejects_negative(monkeypatch: pytest.MonkeyPatch)
("disabled", None),
],
)
[email protected]("setting", ["TASK_V3_DEFAULT_ENGINE_WORKFLOW_CUTOFF", "TASK_V3_CHOSEN_ENGINE_CUTOFF"])
def test_task_v3_default_engine_workflow_cutoff_env_values(
- monkeypatch: pytest.MonkeyPatch, raw_value: str, expected: datetime | None
+ monkeypatch: pytest.MonkeyPatch, setting: str, raw_value: str, expected: datetime | None
) -> None:
- monkeypatch.setenv("TASK_V3_DEFAULT_ENGINE_WORKFLOW_CUTOFF", raw_value)
+ monkeypatch.setenv(setting, raw_value)
- assert Settings(_env_file=None).TASK_V3_DEFAULT_ENGINE_WORKFLOW_CUTOFF == expected
+ assert getattr(Settings(_env_file=None), setting) == expected
diff --git a/tests/unit/test_script_service_fallback_engine.py b/tests/unit/test_script_service_fallback_engine.py
index 19410063b..467a6e863 100644
--- a/tests/unit/test_script_service_fallback_engine.py
+++ b/tests/unit/test_script_service_fallback_engine.py
@@ -13,9 +13,10 @@ from unittest.mock import AsyncMock, MagicMock, patch
import pytest
+from skyvern.config import settings
from skyvern.forge.sdk.core.skyvern_context import SkyvernContext
from skyvern.forge.sdk.schemas.tasks import TaskStatus
-from skyvern.forge.sdk.workflow.models.block import TaskBlock
+from skyvern.forge.sdk.workflow.models.block import TaskBlock, takes_default_engine, v3_ab_ineligibility_reason
from skyvern.forge.sdk.workflow.models.parameter import OutputParameter, ParameterType
from skyvern.forge.sdk.workflow.models.workflow import Workflow, WorkflowDefinition
from skyvern.schemas.runs import RunEngine
@@ -23,6 +24,7 @@ from skyvern.schemas.workflows import BlockType
from skyvern.services import script_service
from skyvern.webeye.actions.action_types import ActionType
from skyvern.webeye.actions.actions import Action, ActionStatus
+from tests.unit._workflow_block_engine_fakes import FakeExperimentationProvider, resolve_arm
MODULE = "skyvern.services.script_service"
@@ -39,7 +41,7 @@ def _make_output_parameter(key: str) -> OutputParameter:
)
-def _make_task_block(label: str, engine: RunEngine = RunEngine.skyvern_v1) -> TaskBlock:
+def _make_task_block(label: str, engine: RunEngine | None = RunEngine.skyvern_v1) -> TaskBlock:
return TaskBlock(
label=label,
output_parameter=_make_output_parameter(f"{label}_output"),
@@ -177,6 +179,42 @@ async def test_fallback_respects_explicit_engine_without_lookup() -> None:
assert _fallback_task_block(app).engine == RunEngine.skyvern_v2
+ ("born_at", "expected"),
+ [
+ (datetime(2026, 10, 2, tzinfo=timezone.utc), RunEngine.skyvern_v3),
+ (datetime(2026, 9, 1, tzinfo=timezone.utc), RunEngine.skyvern_v1),
+ ],
+)
+async def test_fallback_of_an_unset_engine_block_follows_the_chosen_engine_cutoff(
+ monkeypatch: pytest.MonkeyPatch, born_at: datetime, expected: RunEngine
+) -> None:
+ # A script run of a workflow born past the cutoff: its cached block's AI fallback must run where
+ # the same block would run uncached, v3, not on the skyvern_v1 an unset engine used to mean.
+ monkeypatch.setattr(settings, "TASK_V3_DEFAULT_ENGINE_WORKFLOW_CUTOFF", None)
+ monkeypatch.setattr(settings, "TASK_V3_CHOSEN_ENGINE_CUTOFF", datetime(2026, 10, 1, tzinfo=timezone.utc))
+ blocks = [_make_task_block("my_block", engine=None)]
+ workflow = _make_workflow(blocks)
+ context = _make_context()
+ await resolve_arm(
+ context,
+ FakeExperimentationProvider(),
+ workflow_run_id="wr_test",
+ ineligibility_reason=v3_ab_ineligibility_reason(blocks, is_script_run=True),
+ takes_default_engine=takes_default_engine(blocks),
+ first_version_created_at=born_at,
+ )
+ app = _make_app(workflow)
+
+ with patch(f"{MODULE}.app", app), patch(f"{MODULE}.skyvern_context.current", return_value=context):
+ await script_service._fallback_to_ai_run(
+ block_type=BlockType.NAVIGATION, cache_key="my_block", prompt="do the thing"
+ )
+
+ assert _fallback_task_block(app).engine == expected
+
+
def test_resolver_finds_loop_nested_block_engine() -> None:
# A cached block inside a for-loop must keep its configured engine on fallback; the lookup
# is recursive (labels are globally unique, nested included).
@@ -191,8 +229,8 @@ def test_resolver_finds_loop_nested_block_engine() -> None:
output_parameter=nested.output_parameter,
)
workflow = _make_workflow([loop])
- assert script_service._resolve_original_block_engine("inner_block", workflow) == RunEngine.skyvern_v3
- assert script_service._resolve_original_block_engine("missing", workflow) is None
+ assert script_service._resolve_original_block_engine("inner_block", workflow, None) == RunEngine.skyvern_v3
+ assert script_service._resolve_original_block_engine("missing", workflow, None) is None
def _make_run_context(values: dict[str, object]) -> MagicMock:
diff --git a/tests/unit/test_validation_without_page_information.py b/tests/unit/test_validation_without_page_information.py
index f89ce8aee..fdabafdaa 100644
--- a/tests/unit/test_validation_without_page_information.py
+++ b/tests/unit/test_validation_without_page_information.py
@@ -85,7 +85,8 @@ def test_converter_defaults_engine_and_max_steps_per_run_when_omitted() -> None:
historical hardcoded step cap of 2 (not the YAML field's own None default)."""
block_yaml = ValidationBlockYAML(label="v", complete_criterion="x")
block = block_yaml_to_block(block_yaml, {"v_output": _output_parameter("v")})
- assert block.engine == RunEngine.skyvern_v1
+ assert block.engine is None
+ assert block.resolve_engine(None) == RunEngine.skyvern_v1
assert block.max_steps_per_run == 2
diff --git a/tests/unit/test_workflow_block_engine.py b/tests/unit/test_workflow_block_engine.py
index 9207ad8c3..837a54f03 100644
--- a/tests/unit/test_workflow_block_engine.py
+++ b/tests/unit/test_workflow_block_engine.py
@@ -13,6 +13,7 @@ from typing import Any
from unittest.mock import AsyncMock, MagicMock, patch
import pytest
+from pydantic import BaseModel, ConfigDict, ValidationError
from skyvern.config import settings
from skyvern.forge import app
@@ -26,6 +27,7 @@ from skyvern.forge.sdk.experimentation.workflow_block_engine import (
WORKFLOW_TASK_V3_AB_FLAG,
WorkflowBlockEngineRouteReason,
_as_utc,
+ effective_default_engine,
resolve_workflow_block_engine_arm,
workflow_block_engine_override,
)
@@ -52,12 +54,14 @@ from skyvern.forge.sdk.workflow.models.block import (
_evaluate_prompt_branch_conditions_batch,
get_all_blocks,
run_is_eligible_for_v3_ab,
+ takes_default_engine,
v3_ab_ineligibility_reason,
)
+from skyvern.forge.sdk.workflow.models.workflow import WorkflowDefinition
from skyvern.forge.sdk.workflow.service import WorkflowService
from skyvern.forge.taskv3.goal_composition import render_block_context
from skyvern.schemas.run_enums import RunEngine
-from skyvern.schemas.workflows import BlockResult, BlockType, WorkflowStatus
+from skyvern.schemas.workflows import BlockResult, BlockType, NavigationBlockYAML, WorkflowStatus
from skyvern.services import script_service
from tests.unit._workflow_block_engine_fakes import (
WORKFLOW_BLOCK_ENGINE_APP_TARGET,
@@ -92,6 +96,7 @@ def _no_ambient_v3_default_cutoff(monkeypatch: pytest.MonkeyPatch) -> None:
# Settings reads .env at import: without pinning it, every test here answers to whatever the
# machine happens to export.
monkeypatch.setattr(settings, "TASK_V3_DEFAULT_ENGINE_WORKFLOW_CUTOFF", None)
+ monkeypatch.setattr(settings, "TASK_V3_CHOSEN_ENGINE_CUTOFF", None)
@pytest.fixture
@@ -375,6 +380,7 @@ async def test_noop_provider_never_queried_and_leaves_engine_unchanged(scoped_co
workflow_status=WorkflowStatus.published,
trigger_type=WorkflowRunTriggerType.api,
ineligibility_reason=None,
+ takes_default_engine=True,
)
spy.assert_not_called()
@@ -804,6 +810,7 @@ async def test_resolver_logs_the_ineligibility_reason(scoped_context: SkyvernCon
workflow_status=WorkflowStatus.published,
trigger_type=WorkflowRunTriggerType.api,
ineligibility_reason=V3AbIneligibleReason.pinned_engine,
+ takes_default_engine=True,
)
assert scoped_context.workflow_block_engine_override is None
@@ -1010,6 +1017,7 @@ async def test_arm_offers_the_billing_tier_to_the_ab_flag_but_not_the_kill_switc
workflow_status=WorkflowStatus.published,
trigger_type=WorkflowRunTriggerType.api,
ineligibility_reason=None,
+ takes_default_engine=True,
)
by_flag = {flag: properties or {} for flag, _distinct_id, properties in provider.calls}
@@ -1042,6 +1050,7 @@ async def test_a_run_that_never_reaches_the_ab_pays_for_no_tier_lookup(scoped_co
workflow_status=WorkflowStatus.published,
trigger_type=WorkflowRunTriggerType.api,
ineligibility_reason=reason,
+ takes_default_engine=True,
)
tier.assert_not_awaited()
assert mock_log.info.call_args.kwargs["billing_tier"] is None
@@ -1065,6 +1074,7 @@ async def test_a_failing_billing_tier_lookup_cannot_decide_the_arm(scoped_contex
workflow_status=WorkflowStatus.published,
trigger_type=WorkflowRunTriggerType.api,
ineligibility_reason=None,
+ takes_default_engine=True,
)
assert scoped_context.workflow_block_engine_override == RunEngine.skyvern_v3
@@ -1395,3 +1405,235 @@ async def test_a_missing_first_version_falls_back_to_the_flag_path(
assert scoped_context.workflow_block_engine_override is None
assert resolution.log["route_reason"] == WorkflowBlockEngineRouteReason.flag_bucket_control
+
+
+CHOSEN_ENGINE_CUTOFF = datetime(2026, 10, 1, tzinfo=UTC)
+
+
+async def _resolve_chosen_engine_run(
+ context: SkyvernContext,
+ monkeypatch: pytest.MonkeyPatch,
+ blocks: list[BaseTaskBlock],
+ *,
+ run_id: str,
+ cutoff: datetime | None,
+ born_at: datetime,
+ flags: dict[str, bool] | None = None,
+ workflow_status: WorkflowStatus = WorkflowStatus.published,
+) -> tuple[FakeExperimentationProvider, Any]:
+ # Eligibility and the unset-engine fact are computed from the blocks by the same helpers the
+ # workflow service calls, so a block-level regression in either reaches these assertions.
+ monkeypatch.setattr(settings, "TASK_V3_CHOSEN_ENGINE_CUTOFF", cutoff)
+ provider = FakeExperimentationProvider(flags if flags is not None else {WORKFLOW_TASK_V3_AB_FLAG: True})
+ resolution = await resolve_arm(
+ context,
+ provider,
+ workflow_run_id=run_id,
+ ineligibility_reason=v3_ab_ineligibility_reason(blocks, is_script_run=False),
+ takes_default_engine=takes_default_engine(blocks),
+ billing_tier=BillingTier.ENTERPRISE,
+ first_version_created_at=born_at,
+ workflow_status=workflow_status,
+ )
+ return provider, resolution
+
+
+_AFTER = CHOSEN_ENGINE_CUTOFF.replace(tzinfo=None) + timedelta(seconds=1)
+_BEFORE = CHOSEN_ENGINE_CUTOFF.replace(tzinfo=None) - timedelta(days=30)
+_R = WorkflowBlockEngineRouteReason
+
+
+ ("cutoff", "born_at", "chosen", "expected_engine", "expected_reason"),
+ [
+ # Cutoff unset or workflow born before it: today's routing. The A/B is at 100% treatment, so an
+ # unset or skyvern_v1 block is rerouted and a v2/v3 pin makes the run ineligible.
+ (None, _AFTER, None, RunEngine.skyvern_v3, _R.flag_bucket_treatment),
+ (None, _AFTER, RunEngine.skyvern_v1, RunEngine.skyvern_v3, _R.flag_bucket_treatment),
+ (None, _AFTER, RunEngine.skyvern_v2, RunEngine.skyvern_v2, _R.ineligible),
+ (None, _AFTER, RunEngine.skyvern_v3, RunEngine.skyvern_v3, _R.ineligible),
+ (CHOSEN_ENGINE_CUTOFF, _BEFORE, None, RunEngine.skyvern_v3, _R.flag_bucket_treatment),
+ (CHOSEN_ENGINE_CUTOFF, _BEFORE, RunEngine.skyvern_v1, RunEngine.skyvern_v3, _R.flag_bucket_treatment),
+ (CHOSEN_ENGINE_CUTOFF, _BEFORE, RunEngine.skyvern_v2, RunEngine.skyvern_v2, _R.ineligible),
+ (CHOSEN_ENGINE_CUTOFF, _BEFORE, RunEngine.skyvern_v3, RunEngine.skyvern_v3, _R.ineligible),
+ # Born at or after the cutoff: an unset engine runs on v3 and a chosen one runs as written.
+ (CHOSEN_ENGINE_CUTOFF, _AFTER, None, RunEngine.skyvern_v3, _R.new_workflow_v3_default),
+ (CHOSEN_ENGINE_CUTOFF, _AFTER, RunEngine.skyvern_v1, RunEngine.skyvern_v1, _R.chosen_engine),
+ (CHOSEN_ENGINE_CUTOFF, _AFTER, RunEngine.skyvern_v2, RunEngine.skyvern_v2, _R.chosen_engine),
+ (CHOSEN_ENGINE_CUTOFF, _AFTER, RunEngine.skyvern_v3, RunEngine.skyvern_v3, _R.chosen_engine),
+ ],
+)
+async def test_chosen_engine_cutoff_matrix(
+ scoped_context: SkyvernContext,
+ monkeypatch: pytest.MonkeyPatch,
+ cutoff: datetime | None,
+ born_at: datetime,
+ chosen: RunEngine | None,
+ expected_engine: RunEngine,
+ expected_reason: WorkflowBlockEngineRouteReason,
+) -> None:
+ block = _make_block(TaskBlock, label="only", engine=chosen)
+ provider, resolution = await _resolve_chosen_engine_run(
+ scoped_context, monkeypatch, [block], run_id="wr_matrix", cutoff=cutoff, born_at=born_at
+ )
+
+ assert block.resolve_engine("wr_matrix") == expected_engine
+ assert resolution.log["route_reason"] == expected_reason
+ if expected_reason in (_R.new_workflow_v3_default, _R.chosen_engine):
+ # Every tier, with no A/B draw and no billing-tier lookup.
+ assert consulted_flags(provider) == [DISABLE_TASK_V3_FLAG]
+ assert resolution.log["billing_tier"] is None
+
+
+async def test_a_post_cutoff_run_honors_each_block_on_its_own(
+ scoped_context: SkyvernContext, monkeypatch: pytest.MonkeyPatch
+) -> None:
+ # The A/B keeps a run single-engine; a chosen-engine run does not, so a v3 pin (which makes the run
+ # A/B-ineligible) must not hold the unset blocks on v1, and a block v3 cannot run stays on v1.
+ unset = _make_block(NavigationBlock, label="unset", navigation_goal="Apply to the job")
+ chose_v1 = _make_block(TaskBlock, label="chose_v1", engine=RunEngine.skyvern_v1)
+ chose_v3 = _make_block(TaskBlock, label="chose_v3", engine=RunEngine.skyvern_v3)
+ unsupported = _make_block(ValidationBlock, label="unsupported", complete_on_download=True)
+ blocks: list[BaseTaskBlock] = [unset, chose_v1, chose_v3, unsupported]
+ _provider, resolution = await _resolve_chosen_engine_run(
+ scoped_context, monkeypatch, blocks, run_id="wr_mixed", cutoff=CHOSEN_ENGINE_CUTOFF, born_at=_AFTER
+ )
+
+ assert [block.resolve_engine("wr_mixed") for block in blocks] == [
+ RunEngine.skyvern_v3,
+ RunEngine.skyvern_v1,
+ RunEngine.skyvern_v3,
+ RunEngine.skyvern_v1,
+ ]
+ assert resolution.log["route_reason"] == _R.new_workflow_v3_default
+
+
[email protected]("chosen", [None, RunEngine.skyvern_v3])
+async def test_kill_switch_wins_over_the_chosen_engine_cutoff(
+ scoped_context: SkyvernContext, monkeypatch: pytest.MonkeyPatch, chosen: RunEngine | None
+) -> None:
+ block = _make_block(TaskBlock, label="only", engine=chosen)
+ _provider, resolution = await _resolve_chosen_engine_run(
+ scoped_context,
+ monkeypatch,
+ [block],
+ run_id="wr_killed",
+ cutoff=CHOSEN_ENGINE_CUTOFF,
+ born_at=_AFTER,
+ flags={WORKFLOW_TASK_V3_AB_FLAG: True, DISABLE_TASK_V3_FLAG: True},
+ )
+
+ assert scoped_context.workflow_block_engine_override is None
+ resolution.birth_reads.assert_not_awaited()
+ if chosen is None:
+ assert block.resolve_engine("wr_killed") == RunEngine.skyvern_v1
+ assert resolution.log["route_reason"] == _R.disabled
+ else:
+ # The pin is recorded as written and the dispatch gate stops it; its label predates this rule.
+ assert block.resolve_engine("wr_killed") == RunEngine.skyvern_v3
+ assert resolution.log["route_reason"] == _R.ineligible
+
+
+async def test_a_per_call_workflow_born_after_the_chosen_engine_cutoff_keeps_ab_routing(
+ scoped_context: SkyvernContext, monkeypatch: pytest.MonkeyPatch
+) -> None:
+ # Every per-call platform workflow is born after any cutoff; enrolling them would move standing
+ # API traffic, so their explicit skyvern_v1 stays routable.
+ block = _make_block(TaskBlock, label="only", engine=RunEngine.skyvern_v1)
+ _provider, resolution = await _resolve_chosen_engine_run(
+ scoped_context,
+ monkeypatch,
+ [block],
+ run_id="wr_per_call",
+ cutoff=CHOSEN_ENGINE_CUTOFF,
+ born_at=_AFTER,
+ workflow_status=WorkflowStatus.auto_generated,
+ )
+
+ assert block.resolve_engine("wr_per_call") == RunEngine.skyvern_v3
+ assert resolution.log["route_reason"] == _R.flag_bucket_treatment
+ resolution.birth_reads.assert_not_awaited()
+
+
+def test_an_omitted_engine_is_unset_and_a_stored_one_is_kept() -> None:
+ omitted = NavigationBlockYAML(label="nav", url="https://example.com", navigation_goal="Go")
+ stored = NavigationBlockYAML.model_validate(
+ {"label": "nav", "url": "https://example.com", "navigation_goal": "Go", "engine": "skyvern-1.0"}
+ )
+
+ assert omitted.engine is None
+ assert stored.engine == RunEngine.skyvern_v1
+ block = _make_block(NavigationBlock, label="nav", navigation_goal="Go")
+ assert block.engine is None
+ assert type(block).model_validate(block.model_dump()).engine is None
+ # Unset routes like skyvern_v1 before the cutoff, so it never reads as a pin.
+ assert v3_ab_ineligibility_reason([block], is_script_run=False) is None
+ assert takes_default_engine([block]) is True
+
+
+def test_an_unset_engine_is_left_out_of_the_stored_definition_so_an_older_image_reads_it() -> None:
+ # The field as the image before SKY-17436 declares it: a stored null fails validation there, so a
+ # pod still on it during a rollout, or after a revert, could not load the workflow.
+ class OlderImageTaskBlock(BaseModel):
+ model_config = ConfigDict(extra="allow")
+ engine: RunEngine = RunEngine.skyvern_v1
+
+ unset = _make_block(NavigationBlock, label="unset", navigation_goal="Go")
+ chosen = _make_block(TaskBlock, label="chosen", engine=RunEngine.skyvern_v3)
+ stored = WorkflowDefinition(parameters=[], blocks=[unset, chosen]).model_dump(mode="json")
+
+ assert "engine" not in stored["blocks"][0]
+ assert [OlderImageTaskBlock.model_validate(b).engine for b in stored["blocks"]] == [
+ RunEngine.skyvern_v1,
+ RunEngine.skyvern_v3,
+ ]
+ reloaded = WorkflowDefinition.model_validate(stored)
+ assert [b.engine for b in reloaded.blocks] == [None, RunEngine.skyvern_v3]
+ with pytest.raises(ValidationError):
+ OlderImageTaskBlock.model_validate({"engine": None})
+
+
+async def test_both_cutoff_rules_share_one_birth_read(
+ scoped_context: SkyvernContext, monkeypatch: pytest.MonkeyPatch, v3_default_cutoff: datetime
+) -> None:
+ # Born between the two cutoffs: the chosen-engine rule reads the birth and declines, then the
+ # self-serve rule needs the same timestamp.
+ monkeypatch.setattr(settings, "TASK_V3_CHOSEN_ENGINE_CUTOFF", CHOSEN_ENGINE_CUTOFF)
+ resolution = await resolve_arm(
+ scoped_context,
+ FakeExperimentationProvider(),
+ workflow_run_id="wr_between",
+ ineligibility_reason=None,
+ billing_tier=BillingTier.SELF_SERVE,
+ first_version_created_at=CHOSEN_ENGINE_CUTOFF - timedelta(days=1),
+ )
+
+ assert resolution.log["route_reason"] == _R.new_self_serve_workflow_default
+ assert resolution.birth_reads.await_count == 1
+
+
+ ("cutoff", "born_at", "expected"),
+ [
+ (None, _AFTER, None),
+ (CHOSEN_ENGINE_CUTOFF, _BEFORE, None),
+ (CHOSEN_ENGINE_CUTOFF, _AFTER, RunEngine.skyvern_v3),
+ ],
+)
+async def test_effective_default_engine_follows_the_chosen_engine_cutoff(
+ monkeypatch: pytest.MonkeyPatch, cutoff: datetime | None, born_at: datetime, expected: RunEngine | None
+) -> None:
+ monkeypatch.setattr(settings, "TASK_V3_CHOSEN_ENGINE_CUTOFF", cutoff)
+ with patch(WORKFLOW_BLOCK_ENGINE_APP_TARGET) as mock_app:
+ mock_app.EXPERIMENTATION_PROVIDER = FakeExperimentationProvider()
+ mock_app.DATABASE.workflows.get_workflow_permanent_id_created_at = AsyncMock(return_value=born_at)
+ assert await effective_default_engine("wpid_1", "org_1") == expected
+ # With no flag provider the resolver never applies the rule, so the editor must not claim it.
+ mock_app.EXPERIMENTATION_PROVIDER = NoOpExperimentationProvider()
+ assert await effective_default_engine("wpid_1", "org_1") is None
diff --git a/tests/unit/workflow/test_cache_invalidation.py b/tests/unit/workflow/test_cache_invalidation.py
index d6711b85b..ea078100f 100644
--- a/tests/unit/workflow/test_cache_invalidation.py
+++ b/tests/unit/workflow/test_cache_invalidation.py
@@ -5,12 +5,19 @@ Verifies that changes to the model field (both at workflow settings level and bl
do not trigger cache invalidation.
"""
-from datetime import datetime, timezone
+from datetime import datetime, timedelta, timezone
+from unittest.mock import AsyncMock, MagicMock, patch
+import pytest
+
+from skyvern.config import settings
from skyvern.forge.sdk.workflow.models.block import BlockType, CodeBlock, CodeBlockStep, TaskBlock
from skyvern.forge.sdk.workflow.models.parameter import OutputParameter, ParameterType
-from skyvern.forge.sdk.workflow.models.workflow import WorkflowDefinition
-from skyvern.forge.sdk.workflow.service import _get_workflow_definition_core_data
+from skyvern.forge.sdk.workflow.models.workflow import Workflow, WorkflowDefinition
+from skyvern.forge.sdk.workflow.service import WorkflowService, _get_workflow_definition_core_data
+from skyvern.forge.sdk.workflow.workflow_definition_converter import convert_workflow_definition
+from skyvern.schemas.run_enums import RunEngine
+from skyvern.schemas.workflows import WorkflowDefinitionYAML
from skyvern.webeye.actions.action_types import ActionType
@@ -215,3 +222,104 @@ class TestCacheInvalidation:
# These should be identical (timestamps and IDs are excluded)
assert core_data1 == core_data2, "Timestamps and IDs should be excluded from comparison"
+
+
+_TASK_BLOCK_LABELS = ("open", "approve", "nav")
+_CUTOFF = datetime(2026, 10, 1, tzinfo=timezone.utc)
+
+
+def _yaml_definition(nav_engine: str | None) -> WorkflowDefinitionYAML:
+ nav: dict = {"block_type": "navigation", "label": "nav", "url": "https://example.com", "navigation_goal": "Go"}
+ if nav_engine is not None:
+ nav["engine"] = nav_engine
+ return WorkflowDefinitionYAML.model_validate(
+ {
+ "parameters": [],
+ "blocks": [
+ {"block_type": "goto_url", "label": "open", "url": "https://example.com"},
+ {"block_type": "human_interaction", "label": "approve", "recipients": ["[email protected]"]},
+ nav,
+ ],
+ }
+ )
+
+
+def _as_the_previous_image_stored_it(definition: WorkflowDefinition) -> dict:
+ # Before SKY-17436 every task block's engine defaulted to skyvern-1.0 and was always written.
+ stored = definition.model_dump(mode="json")
+ for block in stored["blocks"]:
+ if block["label"] in _TASK_BLOCK_LABELS and block.get("engine") is None:
+ block["engine"] = RunEngine.skyvern_v1.value
+ return stored
+
+
+async def _resave(
+ stored: dict, resaved: WorkflowDefinition, monkeypatch: pytest.MonkeyPatch, *, born_at: datetime
+) -> AsyncMock:
+ """Save ``resaved`` over a workflow whose previous version holds ``stored``; returns the cache clear."""
+ monkeypatch.setattr(settings, "TASK_V3_CHOSEN_ENGINE_CUTOFF", _CUTOFF)
+ now = datetime.now(timezone.utc)
+ previous = Workflow(
+ workflow_id="w_1",
+ organization_id="o_1",
+ title="t",
+ workflow_permanent_id="wpid_1",
+ version=1,
+ is_saved_task=False,
+ workflow_definition=WorkflowDefinition.model_validate(stored),
+ created_at=now,
+ modified_at=now,
+ )
+ service = WorkflowService()
+ database = MagicMock()
+ database.workflows.get_workflow_by_permanent_id = AsyncMock(return_value=previous)
+ with (
+ patch("skyvern.forge.sdk.workflow.service.app") as mock_app,
+ patch("skyvern.forge.sdk.experimentation.workflow_block_engine.app") as engine_app,
+ patch.object(service, "_partition_cached_blocks", AsyncMock(return_value=([MagicMock()], []))),
+ patch.object(service, "_clear_cached_block_groups", AsyncMock()) as clear_groups,
+ ):
+ mock_app.DATABASE = database
+ engine_app.DATABASE.workflows.get_workflow_permanent_id_created_at = AsyncMock(return_value=born_at)
+ await service.maybe_delete_cached_code(
+ previous.model_copy(update={"version": 2}), workflow_definition=resaved, organization_id="o_1"
+ )
+ return clear_groups
+
+
+ "nav_engine",
+ [
+ # The editor, which sends the stored skyvern-1.0 back.
+ RunEngine.skyvern_v1.value,
+ # An API caller re-sending the YAML it created the workflow with, which omitted the engine.
+ None,
+ ],
+)
+async def test_an_unchanged_resave_of_a_previously_stored_definition_keeps_its_cached_scripts(
+ monkeypatch: pytest.MonkeyPatch, nav_engine: str | None
+) -> None:
+ resaved = convert_workflow_definition(_yaml_definition(nav_engine), "w_1")
+ stored = _as_the_previous_image_stored_it(resaved)
+
+ clear_groups = await _resave(stored, resaved, monkeypatch, born_at=_CUTOFF - timedelta(days=30))
+
+ clear_groups.assert_not_awaited()
+ if nav_engine is not None:
+ # What the editor saves is what was stored, in every engine field.
+ assert resaved.model_dump(mode="json") == stored
+
+
+async def test_past_the_cutoff_switching_a_legacy_pin_to_default_is_a_change(monkeypatch: pytest.MonkeyPatch) -> None:
+ # Unset runs on v3 there and skyvern-1.0 is an explicit pin, so the cached script must be cleared;
+ # the engine-inert blocks around it still compare equal.
+ stored = convert_workflow_definition(_yaml_definition(RunEngine.skyvern_v1.value), "w_1").model_dump(mode="json")
+ resaved = convert_workflow_definition(_yaml_definition(None), "w_1")
+ born_after = _CUTOFF + timedelta(days=1)
+
+ clear_groups = await _resave(stored, resaved, monkeypatch, born_at=born_after)
+
+ clear_groups.assert_awaited_once()
+ assert clear_groups.await_args.kwargs["plan"].label == "nav"
diff --git a/tests/unit/workflow/test_validation_action_yaml_field_passthrough.py b/tests/unit/workflow/test_validation_action_yaml_field_passthrough.py
index 60dd2b6cc..f68b3ff7e 100644
--- a/tests/unit/workflow/test_validation_action_yaml_field_passthrough.py
+++ b/tests/unit/workflow/test_validation_action_yaml_field_passthrough.py
@@ -80,7 +80,9 @@ def test_yaml_import_defaults_when_new_keys_omitted() -> None:
validation_block = blocks["v"]
assert isinstance(validation_block, ValidationBlock)
- assert validation_block.engine == RunEngine.skyvern_v1
+ # Stored unset, and resolved exactly like the old skyvern_v1 default outside a chosen-engine run.
+ assert validation_block.engine is None
+ assert validation_block.resolve_engine(None) == RunEngine.skyvern_v1
assert validation_block.max_steps_per_run == 2
action_block = blocks["a"]