Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
Honor the chosen block engine for workflows created after a cutoff (backend) (#8732)
pedrohsdba9bed62b14d2
diff --git a/docs/api-reference/openapi.json b/docs/api-reference/openapi.jsonindex e1c975fd0..e4fc9b113 100644--- a/docs/api-reference/openapi.json+++ b/docs/api-reference/openapi.json@@ -8942,8 +8942,14 @@"default": ""},"engine": {- "$ref": "#/components/schemas/RunEngine",- "default": "skyvern-1.0"+ "anyOf": [+ {+ "$ref": "#/components/schemas/RunEngine"+ },+ {+ "type": "null"+ }+ ]},"complete_criterion": {"anyOf": [@@ -9257,8 +9263,14 @@"default": ""},"engine": {- "$ref": "#/components/schemas/RunEngine",- "default": "skyvern-1.0"+ "anyOf": [+ {+ "$ref": "#/components/schemas/RunEngine"+ },+ {+ "type": "null"+ }+ ]},"navigation_goal": {"anyOf": [@@ -14455,8 +14467,14 @@"default": ""},"engine": {- "$ref": "#/components/schemas/RunEngine",- "default": "skyvern-1.0"+ "anyOf": [+ {+ "$ref": "#/components/schemas/RunEngine"+ },+ {+ "type": "null"+ }+ ]},"complete_criterion": {"anyOf": [@@ -14792,8 +14810,14 @@"default": ""},"engine": {- "$ref": "#/components/schemas/RunEngine",- "default": "skyvern-1.0"+ "anyOf": [+ {+ "$ref": "#/components/schemas/RunEngine"+ },+ {+ "type": "null"+ }+ ]},"data_schema": {"anyOf": [@@ -15205,8 +15229,14 @@"default": ""},"engine": {- "$ref": "#/components/schemas/RunEngine",- "default": "skyvern-1.0"+ "anyOf": [+ {+ "$ref": "#/components/schemas/RunEngine"+ },+ {+ "type": "null"+ }+ ]},"complete_criterion": {"anyOf": [@@ -15742,8 +15772,14 @@"default": ""},"engine": {- "$ref": "#/components/schemas/RunEngine",- "default": "skyvern-1.0"+ "anyOf": [+ {+ "$ref": "#/components/schemas/RunEngine"+ },+ {+ "type": "null"+ }+ ]},"error_code_mapping": {"anyOf": [@@ -18630,7 +18666,14 @@"default": ""},"engine": {- "$ref": "#/components/schemas/RunEngine",+ "anyOf": [+ {+ "$ref": "#/components/schemas/RunEngine"+ },+ {+ "type": "null"+ }+ ],"default": "skyvern-1.0"},"complete_criterion": {@@ -19443,8 +19486,14 @@"default": ""},"engine": {- "$ref": "#/components/schemas/RunEngine",- "default": "skyvern-1.0"+ "anyOf": [+ {+ "$ref": "#/components/schemas/RunEngine"+ },+ {+ "type": "null"+ }+ ]},"complete_criterion": {"anyOf": [@@ -19748,8 +19797,14 @@"default": ""},"engine": {- "$ref": "#/components/schemas/RunEngine",- "default": "skyvern-1.0"+ "anyOf": [+ {+ "$ref": "#/components/schemas/RunEngine"+ },+ {+ "type": "null"+ }+ ]},"navigation_goal": {"anyOf": [@@ -20257,8 +20312,14 @@"default": ""},"engine": {- "$ref": "#/components/schemas/RunEngine",- "default": "skyvern-1.0"+ "anyOf": [+ {+ "$ref": "#/components/schemas/RunEngine"+ },+ {+ "type": "null"+ }+ ]},"complete_criterion": {"anyOf": [@@ -20555,8 +20616,14 @@"default": ""},"engine": {- "$ref": "#/components/schemas/RunEngine",- "default": "skyvern-1.0"+ "anyOf": [+ {+ "$ref": "#/components/schemas/RunEngine"+ },+ {+ "type": "null"+ }+ ]},"error_code_mapping": {"anyOf": [@@ -24442,8 +24509,14 @@"default": ""},"engine": {- "$ref": "#/components/schemas/RunEngine",- "default": "skyvern-1.0"+ "anyOf": [+ {+ "$ref": "#/components/schemas/RunEngine"+ },+ {+ "type": "null"+ }+ ]},"complete_criterion": {"anyOf": [@@ -24742,8 +24815,14 @@"default": ""},"engine": {- "$ref": "#/components/schemas/RunEngine",- "default": "skyvern-1.0"+ "anyOf": [+ {+ "$ref": "#/components/schemas/RunEngine"+ },+ {+ "type": "null"+ }+ ]},"navigation_goal": {"anyOf": [@@ -27396,7 +27475,14 @@"default": ""},"engine": {- "$ref": "#/components/schemas/RunEngine",+ "anyOf": [+ {+ "$ref": "#/components/schemas/RunEngine"+ },+ {+ "type": "null"+ }+ ],"default": "skyvern-1.0"},"complete_criterion": {@@ -27839,8 +27925,14 @@"default": ""},"engine": {- "$ref": "#/components/schemas/RunEngine",- "default": "skyvern-1.0"+ "anyOf": [+ {+ "$ref": "#/components/schemas/RunEngine"+ },+ {+ "type": "null"+ }+ ]},"complete_criterion": {"anyOf": [@@ -28128,8 +28220,14 @@"default": false},"engine": {- "$ref": "#/components/schemas/RunEngine",- "default": "skyvern-1.0"+ "anyOf": [+ {+ "$ref": "#/components/schemas/RunEngine"+ },+ {+ "type": "null"+ }+ ]},"complete_criterion": {"anyOf": [@@ -29561,6 +29659,17 @@"title": "Copilot Authored","default": false},+ "effective_default_engine": {+ "anyOf": [+ {+ "$ref": "#/components/schemas/RunEngine"+ },+ {+ "type": "null"+ }+ ],+ "description": "The engine a task block with no `engine` set runs on in this agent, or null when engine routing decides it. Populated by the detail endpoint only."+ },"original_created_by": {"anyOf": [{diff --git a/skyvern/config.py b/skyvern/config.pyindex bcaba82f0..fd49b4f31 100644--- a/skyvern/config.py+++ b/skyvern/config.py@@ -591,6 +591,11 @@ class Settings(BaseSettings):# which covers the job recipe endpoints because those build a published definition. Enrolled runs# are not randomized, so every per-arm read must exclude them by route_reason.TASK_V3_DEFAULT_ENGINE_WORKFLOW_CUTOFF: datetime | None = None+ # Workflows whose permanent id was born at or after this instant, in every billing tier, honor each+ # task block's chosen engine and run a block with no engine on Task V3, outside WORKFLOW_TASK_V3_AB.+ # Excludes the same per-call platform workflows as the cutoff above. None disables the rule, as does+ # having no flag provider; DISABLE_TASK_V3 still wins. A naive value is read as UTC.+ TASK_V3_CHOSEN_ENGINE_CUTOFF: datetime | None = None# VOLCENGINE (Doubao)ENABLE_VOLCENGINE: bool = False@@ -1078,7 +1083,7 @@ class Settings(BaseSettings):return Nonereturn value- @field_validator("TASK_V3_DEFAULT_ENGINE_WORKFLOW_CUTOFF", mode="before")+ @field_validator("TASK_V3_DEFAULT_ENGINE_WORKFLOW_CUTOFF", "TASK_V3_CHOSEN_ENGINE_CUTOFF", mode="before")@classmethoddef _task_v3_default_engine_workflow_cutoff_off_sentinels(cls, value: Any) -> Any:# This setting is the rule's settings-side kill path, and blanking an already-set env var isdiff --git a/skyvern/forge/sdk/experimentation/workflow_block_engine.py b/skyvern/forge/sdk/experimentation/workflow_block_engine.pyindex 4fdca7ee5..572695789 100644--- a/skyvern/forge/sdk/experimentation/workflow_block_engine.py+++ b/skyvern/forge/sdk/experimentation/workflow_block_engine.py@@ -46,6 +46,11 @@ class WorkflowBlockEngineRouteReason(StrEnum):flag_bucket_treatment = "flag_bucket_treatment"flag_bucket_control = "flag_bucket_control"new_self_serve_workflow_default = "new_self_serve_workflow_default"+ # A workflow born at or after TASK_V3_CHOSEN_ENGINE_CUTOFF, never randomized: at least one block+ # left its engine unset and runs on v3 (``new_workflow_v3_default``), or every block's engine was+ # chosen and is honored as written (``chosen_engine``).+ new_workflow_v3_default = "new_workflow_v3_default"+ chosen_engine = "chosen_engine"ineligible = "ineligible"disabled = "disabled"flag_undefined = "flag_undefined"@@ -80,6 +85,8 @@ class WorkflowBlockEngineArmDecision:route_reason: WorkflowBlockEngineRouteReason | None = Nonebilling_tier: BillingTier | None = None+ # An explicit skyvern_v1 on a block is a pin, not a routable default (see Block.resolve_engine).+ honors_chosen_engine: bool = FalseNO_ARM_DECISION = WorkflowBlockEngineArmDecision()@@ -144,15 +151,14 @@ def _as_utc(value: datetime) -> datetime:return value.replace(tzinfo=UTC) if value.tzinfo is None else value.astimezone(UTC)-async def _workflow_is_new_for_v3_default(workflow_permanent_id: str | None, organization_id: str | None) -> bool:- """Whether this permanent id was born at or after the v3-default cutoff.+async def _workflow_birth(workflow_permanent_id: str | None, organization_id: str | None) -> datetime | None:+ """When this permanent id was born, in UTC, or None when that could not be read.- Never raises and never answers True on a read it could not complete: the rule can only add- treatment, so any doubt has to leave the run on the arm the A/B would have given it.+ Never raises, and None satisfies no cutoff: both cutoff rules take a run out of the A/B, so any+ doubt has to leave the run on the arm the A/B would have given it."""- cutoff = settings.TASK_V3_DEFAULT_ENGINE_WORKFLOW_CUTOFF- if cutoff is None or not workflow_permanent_id or not organization_id:- return False+ if not workflow_permanent_id or not organization_id:+ return Nonetry:# The earliest version's timestamp, deleted versions included, is when the permanent id was# born. Reading the version this run executes would instead enrol every long-lived workflow@@ -160,18 +166,30 @@ async def _workflow_is_new_for_v3_default(workflow_permanent_id: str | None, orgborn_at = await app.DATABASE.workflows.get_workflow_permanent_id_created_at(workflow_permanent_id, organization_id)- if born_at is None:- return False- # Inside the try with the read: a rule that raised here would reach the resolver's catch-all- # and send the run to control, which is a different arm than the A/B would have given it.- return _as_utc(born_at) >= _as_utc(cutoff)+ # Inside the try with the read: a conversion that raised here would reach the resolver's+ # catch-all and send the run to control, a different arm than the A/B would have given it.+ return None if born_at is None else _as_utc(born_at)except Exception:LOG.warning(- "Failed to read the workflow's birth timestamp for the Task V3 default rule",+ "Failed to read the workflow's birth timestamp for a Task V3 cutoff rule",workflow_permanent_id=workflow_permanent_id,exc_info=True,)- return False+ return None+++def _born_at_or_after(born_at: datetime | None, cutoff: datetime | None) -> bool:+ return born_at is not None and cutoff is not None and born_at >= _as_utc(cutoff)+++def _is_kept_workflow_run(workflow_status: WorkflowStatus, trigger_type: WorkflowRunTriggerType | None) -> bool:+ # An auto_generated running version is a per-call workflow the login, download_files, credential+ # test-login and SDK endpoints mint and run in one request, and enrolling those would move standing+ # API traffic rather than the workflows a customer keeps; the birth version's status would be the+ # wrong test, because the prompt box writes version 1 as auto_generated and the editor save that+ # keeps it writes the next as published. The job-recipe endpoints are per-call too but build a+ # published definition, so only their trigger kind excludes them.+ return workflow_status != WorkflowStatus.auto_generated and trigger_type not in PER_CALL_WORKFLOW_RUN_TRIGGER_TYPESasync def _ab_flag_puts_run_in_treatment(@@ -213,6 +231,7 @@ async def resolve_workflow_block_engine_arm(workflow_status: WorkflowStatus,trigger_type: WorkflowRunTriggerType | None,ineligibility_reason: V3AbIneligibleReason | None,+ takes_default_engine: bool | None,) -> None:"""Resolve the workflow-block engine A/B once at execution start and pin the arm on the context.@@ -228,6 +247,13 @@ async def resolve_workflow_block_engine_arm(consulted, so those runs are marked ``new_self_serve_workflow_default`` for every per-arm read toexclude. No flag gates the rule: ``DISABLE_TASK_V3`` or unsetting the cutoff turns it off.+ A workflow of any tier born at or after ``settings.TASK_V3_CHOSEN_ENGINE_CUTOFF`` skips the A/B+ and the rule above: every block's chosen engine is honored and an unset one runs on v3.+ ``takes_default_engine`` (see ``block.takes_default_engine``) says whether any block left its+ engine unset, or is None when the run has no engine to choose. The rule does not require A/B+ eligibility, since a pinned block no longer keeps the run's unset blocks on v1, and so covers a+ script run's uncached blocks and AI fallback.+``workflow_status`` is the status of the version this run executes and ``trigger_type`` is howthe run was launched. Together they are what separates a workflow a customer keeps from theper-call throwaways several endpoints mint and run in one request: most of those run while still@@ -247,11 +273,27 @@ async def resolve_workflow_block_engine_arm(if context.workflow_block_engine_resolved_run_id == workflow_run_id:returnoverride: RunEngine | None = None+ honors_chosen_engine = Falserun_is_eligible = ineligibility_reason is Nonebilling_tier: BillingTier | None = Noneroute_reason = WorkflowBlockEngineRouteReason.ineligible+ chosen_engine_candidate = (+ settings.TASK_V3_CHOSEN_ENGINE_CUTOFF is not None+ and takes_default_engine is not None+ and _is_kept_workflow_run(workflow_status, trigger_type)+ )+ # Both cutoff rules compare the same birth timestamp, so it is read at most once per run.+ birth_reads: list[datetime | None] = []++ async def born_at_or_after(cutoff: datetime | None) -> bool:+ if cutoff is None:+ return False+ if not birth_reads:+ birth_reads.append(await _workflow_birth(workflow_permanent_id, organization_id))+ return _born_at_or_after(birth_reads[0], cutoff)+try:- if run_is_eligible:+ if run_is_eligible or chosen_engine_candidate:# The kill switch, shared with the dispatch gate via task_v3_disabled so both# evaluations use the same cache key, wins over the experiment and over the# new-workflow default below. A kill flipped mid-run still takes effect at dispatch@@ -260,27 +302,27 @@ async def resolve_workflow_block_engine_arm(# when someone flips this, and this lock is held throughout, so the kill switch must# not queue behind that call.if await task_v3_disabled(workflow_run_id, organization_id):- route_reason = WorkflowBlockEngineRouteReason.disabled- else:+ if run_is_eligible:+ route_reason = WorkflowBlockEngineRouteReason.disabled+ elif chosen_engine_candidate and await born_at_or_after(settings.TASK_V3_CHOSEN_ENGINE_CUTOFF):+ honors_chosen_engine = True+ if takes_default_engine:+ override = RunEngine.skyvern_v3+ route_reason = WorkflowBlockEngineRouteReason.new_workflow_v3_default+ else:+ route_reason = WorkflowBlockEngineRouteReason.chosen_engine+ elif run_is_eligible:# billing_tier rides along so a release condition can hold enterprise and# self-serve at different percentages. Deliberately not passed to# task_v3_disabled: that flag's two callers must build identical provider cache# keys, see its docstring.billing_tier = await _billing_tier_for_arm(organization_id)# The tier, the status and the trigger are checked first, so anything that- # cannot use the rule pays for no workflow read. An auto_generated running- # version is a per-call workflow the login, download_files, credential- # test-login and SDK endpoints mint and run in one request, and enrolling those- # would move standing API traffic rather than the workflows a customer keeps;- # the birth version's status would be the wrong test, because the prompt box- # writes version 1 as auto_generated and the editor save that keeps it writes- # the next as published. The job-recipe endpoints are per-call too but build a- # published definition, so only their trigger kind excludes them.+ # cannot use the rule pays for no workflow read.takes_new_workflow_default = (billing_tier == BillingTier.SELF_SERVE- and workflow_status != WorkflowStatus.auto_generated- and trigger_type not in PER_CALL_WORKFLOW_RUN_TRIGGER_TYPES- and await _workflow_is_new_for_v3_default(workflow_permanent_id, organization_id)+ and _is_kept_workflow_run(workflow_status, trigger_type)+ and await born_at_or_after(settings.TASK_V3_DEFAULT_ENGINE_WORKFLOW_CUTOFF))if takes_new_workflow_default:# A workflow born at or after the cutoff runs its task blocks on v3 by@@ -312,10 +354,12 @@ async def resolve_workflow_block_engine_arm(exc_info=True,)override = None+ honors_chosen_engine = Falseroute_reason = WorkflowBlockEngineRouteReason.flag_errordecision = WorkflowBlockEngineArmDecision(route_reason=route_reason,billing_tier=billing_tier,+ honors_chosen_engine=honors_chosen_engine,)context.workflow_block_engine_override = overridecontext.workflow_block_engine_arm_decision = decision@@ -351,6 +395,30 @@ def workflow_block_engine_override(workflow_run_id: str | None) -> RunEngine | Nreturn context.workflow_block_engine_override+def run_honors_chosen_engine(workflow_run_id: str | None) -> bool:+ if not workflow_run_id:+ return False+ context = skyvern_context.current()+ if context is None or context.workflow_block_engine_resolved_run_id != workflow_run_id:+ return False+ decision = context.workflow_block_engine_arm_decision+ return decision is not None and decision.honors_chosen_engine+++async def effective_default_engine(workflow_permanent_id: str, organization_id: str) -> RunEngine | None:+ """The engine a block with no chosen engine runs on in this workflow, or None when routing decides it.++ Mirrors the resolver's chosen-engine rule for the editor; the per-call exclusions do not apply to+ a workflow someone opens there.+ """+ cutoff = settings.TASK_V3_CHOSEN_ENGINE_CUTOFF+ if cutoff is None or isinstance(app.EXPERIMENTATION_PROVIDER, NoOpExperimentationProvider):+ return None+ if _born_at_or_after(await _workflow_birth(workflow_permanent_id, organization_id), cutoff):+ return RunEngine.skyvern_v3+ return None++def resolved_workflow_block_engine_arm_attribution(workflow_run_id: str | None) -> WorkflowBlockEngineArmAttribution:"""The arm and the routing facts finalize-time telemetry can attribute to this run.diff --git a/skyvern/forge/sdk/routes/agent_protocol.py b/skyvern/forge/sdk/routes/agent_protocol.pyindex 891bdaadb..311d3fd7e 100644--- a/skyvern/forge/sdk/routes/agent_protocol.py+++ b/skyvern/forge/sdk/routes/agent_protocol.py@@ -78,6 +78,7 @@ from skyvern.forge.sdk.db.repositories.tags import (from skyvern.forge.sdk.db.repositories.workflows import WorkflowCreationLockTimeoutfrom skyvern.forge.sdk.enterprise_features import collect_enterprise_gated_run_featuresfrom skyvern.forge.sdk.executor.factory import AsyncExecutorFactory+from skyvern.forge.sdk.experimentation.workflow_block_engine import effective_default_enginefrom skyvern.forge.sdk.models import Stepfrom skyvern.forge.sdk.routes.code_samples import (BULK_CANCEL_RUNS_CODE_SAMPLE_PYTHON,@@ -5382,6 +5383,9 @@ async def get_workflow(workflow_permanent_id=workflow_permanent_id,organization_id=current_org.organization_id,)+ workflow.effective_default_engine = await effective_default_engine(+ workflow_permanent_id, current_org.organization_id+ )return workflowdiff --git a/skyvern/forge/sdk/workflow/models/block.py b/skyvern/forge/sdk/workflow/models/block.pyindex 3e2f0c391..6a708acfe 100644--- a/skyvern/forge/sdk/workflow/models/block.py+++ b/skyvern/forge/sdk/workflow/models/block.py@@ -179,7 +179,10 @@ from skyvern.forge.sdk.db.exceptions import NotFoundErrorfrom skyvern.forge.sdk.db.id import generate_action_idfrom skyvern.forge.sdk.experimentation.code_block_ai_fallback import code_block_ai_fallback_flag_enabledfrom skyvern.forge.sdk.experimentation.llm_prompt_config import get_llm_handler_for_prompt_type-from skyvern.forge.sdk.experimentation.workflow_block_engine import workflow_block_engine_override+from skyvern.forge.sdk.experimentation.workflow_block_engine import (+ run_honors_chosen_engine,+ workflow_block_engine_override,+)from skyvern.forge.sdk.forge_log import exception_log_fieldsfrom skyvern.forge.sdk.models import Step, StepStatusfrom skyvern.forge.sdk.schemas.files import FileInfo@@ -1879,11 +1882,17 @@ def _should_skip_retry_on_anti_bot_detection(task: Task) -> bool:return False+def _engine_is_unset(engine: RunEngine | None) -> bool:+ return engine is None++class BaseTaskBlock(Block):task_type: str = TaskType.generalurl: str | None = Nonetitle: str = ""- engine: RunEngine = RunEngine.skyvern_v1+ # Left out of the dump when unset, so a stored definition stays readable by an image whose engine+ # field still rejects null (a rollout or a revert).+ engine: RunEngine | None = Field(default=None, exclude_if=_engine_is_unset)complete_criterion: str | None = Nonecomplete_criterion_is_untrusted: bool = Falseterminate_criterion: str | None = None@@ -1974,19 +1983,25 @@ class BaseTaskBlock(Block):Both the persisted workflow_run_blocks.engine and the execute_step dispatch read this, sothe recorded engine cannot disagree with the one that ran. A block pinned to a non-defaultengine is honored as-authored, and a block the eligibility check never saw is left alone;- neither is ever rerouted.+ neither is ever rerouted. An unset engine routes like skyvern_v1, except in a run that honors the+ chosen engine, where an explicit skyvern_v1 is a pin too."""+ declared = self.engine or RunEngine.skyvern_v1if (- self.engine != RunEngine.skyvern_v1- or self._exclude_from_engine_ab+ self._exclude_from_engine_ab# Mirrors run_is_eligible_for_v3_ab: a block eligibility skipped as engine-inert must- # not be labeled v3 here either, or its row claims an engine that never ran. It does not- # re-check _task_block_supports_v3 because run-level eligibility already rejected the- # whole run if any block failed it; loosening that predicate means revisiting this.+ # not be labeled v3 here either, or its row claims an engine that never ran.or self.block_type in _ENGINE_INERT_BLOCK_TYPES+ # Redundant for an A/B run, whose eligibility already rejected any such block; a+ # chosen-engine run skips that check, so a block v3 cannot execute stays off the override.+ or not _task_block_supports_v3(self)+ ):+ return declared+ if self.engine is not None and (+ self.engine != RunEngine.skyvern_v1 or run_honors_chosen_engine(workflow_run_id)):return self.engine- return workflow_block_engine_override(workflow_run_id) or self.engine+ return workflow_block_engine_override(workflow_run_id) or declareddef get_all_parameters(self,@@ -14427,6 +14442,9 @@ class HumanInteractionBlock(BaseTaskBlock):# There is a mypy bug with Literal. Without the type: ignore, mypy will raise an error:# Parameter 1 of Literal[...] cannot be of type "Any"block_type: Literal[BlockType.HUMAN_INTERACTION] = BlockType.HUMAN_INTERACTION # type: ignore+ # Engine-inert, and its YAML has no engine: keeping the stored value makes an unedited re-save+ # write the definition it read, so the cached script is not cleared.+ engine: RunEngine | None = Field(default=RunEngine.skyvern_v1, exclude_if=_engine_is_unset)instructions: str = "Please review and approve or reject to continue the workflow."positive_descriptor: str = "Approve"@@ -15390,6 +15408,8 @@ class UrlBlock(BaseTaskBlock):# Parameter 1 of Literal[...] cannot be of type "Any"block_type: Literal[BlockType.GOTO_URL] = BlockType.GOTO_URL # type: ignoreurl: str+ # Engine-inert; see HumanInteractionBlock.engine.+ engine: RunEngine | None = Field(default=RunEngine.skyvern_v1, exclude_if=_engine_is_unset)class TaskV2Block(Block):@@ -18946,7 +18966,7 @@ def v3_ab_ineligibility_reason(blocks: list[BlockTypeVar], *, is_script_run: boocontinueif block.block_type in _ENGINE_INERT_BLOCK_TYPES:continue- if block.engine != RunEngine.skyvern_v1:+ if block.engine not in (None, RunEngine.skyvern_v1):return V3AbIneligibleReason.pinned_engineif not _task_block_supports_v3(block):return V3AbIneligibleReason.unsupported_block@@ -18958,6 +18978,31 @@ def v3_ab_ineligibility_reason(blocks: list[BlockTypeVar], *, is_script_run: booreturn None+def takes_default_engine(blocks: list[BlockTypeVar]) -> bool | None:+ """Whether any block of the run leaves its engine unchosen; None when the run has no engine to choose.++ ``blocks`` is the flattened definition, as for v3_ab_ineligibility_reason. A script run is answered+ too: its uncached blocks and its AI fallback run on the engine this decides. A prompt-criteria+ condition counts as unchosen: its synthetic extraction block has no engine an author can set.+ """+ has_engine_block = False+ for block in blocks:+ if isinstance(block, ConditionalBlock):+ if any(isinstance(branch.criteria, PromptBranchCriteria) for branch in block.branch_conditions):+ return True+ continue+ if isinstance(block, WhileLoopBlock):+ if isinstance(block.condition, PromptBranchCriteria):+ return True+ continue+ if not isinstance(block, BaseTaskBlock) or block.block_type in _ENGINE_INERT_BLOCK_TYPES:+ continue+ if block.engine is None and _task_block_supports_v3(block):+ return True+ has_engine_block = True+ return False if has_engine_block else None++def run_is_eligible_for_v3_ab(blocks: list[BlockTypeVar], *, is_script_run: bool) -> bool:"""Whether a whole workflow run may be rerouted onto v3 by the A/B; see v3_ab_ineligibility_reason."""return v3_ab_ineligibility_reason(blocks, is_script_run=is_script_run) is Nonediff --git a/skyvern/forge/sdk/workflow/models/workflow.py b/skyvern/forge/sdk/workflow/models/workflow.pyindex dd8d07137..68852ad46 100644--- a/skyvern/forge/sdk/workflow/models/workflow.py+++ b/skyvern/forge/sdk/workflow/models/workflow.py@@ -32,6 +32,7 @@ from skyvern.forge.sdk.workflow.models.validators import (normalize_run_metadata,normalize_run_with,)+from skyvern.schemas.run_enums import RunEnginefrom skyvern.schemas.runs import (BROWSER_ADDRESS_SERVER_ASSIGNED_CONTEXT_KEY,BROWSER_TYPE_ATTACH_CONFLICT_MESSAGE,@@ -260,6 +261,11 @@ class Workflow(BaseModel):# Lineage-derived (any version copilot-stamped); populated by the detail GET route only —# user saves re-stamp created_by/edited_by, so the current version alone is not durable.copilot_authored: bool = False+ effective_default_engine: RunEngine | None = Field(+ default=None,+ description="The engine a task block with no `engine` set runs on in this agent, or null when "+ "engine routing decides it. Populated by the detail endpoint only.",+ )original_created_by: str | None = Field(default=None,description="Who created the agent's first version. Populated by the list endpoint only.",diff --git a/skyvern/forge/sdk/workflow/service.py b/skyvern/forge/sdk/workflow/service.pyindex e098fafcb..fcb01aedb 100644--- a/skyvern/forge/sdk/workflow/service.py+++ b/skyvern/forge/sdk/workflow/service.py@@ -120,6 +120,7 @@ from skyvern.forge.sdk.experimentation.transient_ui_capture import resolve_transfrom skyvern.forge.sdk.experimentation.workflow_block_engine import (ARM_ATTRIBUTION_LOST,WorkflowBlockEngineArmAttribution,+ effective_default_engine,engine_arm_log_value,resolve_workflow_block_engine_arm,resolved_workflow_block_engine_arm_attribution,@@ -167,6 +168,7 @@ from skyvern.forge.sdk.workflow.exceptions import (WorkflowVersionConflict,)from skyvern.forge.sdk.workflow.models.block import (+ _ENGINE_INERT_BLOCK_TYPES,BaseTaskBlock,Block,BlockTypeVar,@@ -188,6 +190,7 @@ from skyvern.forge.sdk.workflow.models.block import (compute_conditional_scopes,get_all_blocks,resolve_conditional_merge_edges,+ takes_default_engine,v3_ab_ineligibility_reason,)from skyvern.forge.sdk.workflow.models.parameter import (@@ -1423,7 +1426,9 @@ class ReusedSessionBelowLifetimeFloor(Exception):self.shortfall = shortfall-def _get_workflow_definition_core_data(workflow_definition: WorkflowDefinition) -> dict[str, Any]:+def _get_workflow_definition_core_data(+ workflow_definition: WorkflowDefinition, *, unset_engine_is_v1: bool = True+) -> dict[str, Any]:"""This function dumps the workflow definition and removes the irrelevant data to the definition, like created_at and modified_at fields inside:- list of blocks@@ -1472,6 +1477,15 @@ def _get_workflow_definition_core_data(workflow_definition: WorkflowDefinition)if current_obj.get("block_type") == BlockType.CODE.value:for field in code_block_annotation_fields:current_obj.pop(field, None)+ # A definition stored before the engine became optional holds skyvern-1.0 where a re-save+ # of the same blocks leaves it unset. The two route alike on an engine-inert block and+ # before the chosen-engine cutoff; past it, unset means v3 and skyvern-1.0 is a pin.+ if (+ "block_type" in current_obj+ and current_obj.get("engine") in (None, RunEngine.skyvern_v1.value)+ and (unset_engine_is_v1 or current_obj["block_type"] in _ENGINE_INERT_BLOCK_TYPES)+ ):+ current_obj.pop("engine", None)# Add all nested dictionaries and lists to queue for processingfor value in current_obj.values():@@ -7818,6 +7832,7 @@ class WorkflowService:workflow_status=workflow.status,trigger_type=workflow_run.trigger_type,ineligibility_reason=v3_ab_ineligibility_reason(all_blocks, is_script_run=is_script_run),+ takes_default_engine=takes_default_engine(all_blocks),)else:LOG.warning(@@ -11142,8 +11157,13 @@ class WorkflowService:current_definition: dict[str, Any] = {}new_definition: dict[str, Any] = {}if previous_valid_workflow:- current_definition = _get_workflow_definition_core_data(previous_valid_workflow.workflow_definition)- new_definition = _get_workflow_definition_core_data(workflow_definition)+ unset_engine_is_v1 = await effective_default_engine(workflow.workflow_permanent_id, organization_id) is None+ current_definition = _get_workflow_definition_core_data(+ previous_valid_workflow.workflow_definition, unset_engine_is_v1=unset_engine_is_v1+ )+ new_definition = _get_workflow_definition_core_data(+ workflow_definition, unset_engine_is_v1=unset_engine_is_v1+ )has_changes = current_definition != new_definition# Log definition changes for debugging cache invalidation issuesdiff --git a/skyvern/schemas/workflows.py b/skyvern/schemas/workflows.pyindex 3be75297d..898123dd7 100644--- a/skyvern/schemas/workflows.py+++ b/skyvern/schemas/workflows.py@@ -840,7 +840,7 @@ class TaskBlockYAML(BlockYAML):url: str | None = Nonetitle: str = ""- engine: RunEngine = RunEngine.skyvern_v1+ engine: RunEngine | None = Nonenavigation_goal: str | None = Nonedata_extraction_goal: str | None = Nonedata_schema: dict[str, Any] | list | str | None = None@@ -1245,7 +1245,7 @@ class PDFParserBlockYAML(BlockYAML):class ValidationBlockYAML(BlockYAML):block_type: Literal[BlockType.VALIDATION] = BlockType.VALIDATION # type: ignore- engine: RunEngine = RunEngine.skyvern_v1+ engine: RunEngine | None = Nonecomplete_criterion: str | None = Noneterminate_criterion: str | None = Noneerror_code_mapping: dict[str, str] | None = None@@ -1260,7 +1260,7 @@ class ActionBlockYAML(BlockYAML):url: str | None = Nonetitle: str = ""- engine: RunEngine = RunEngine.skyvern_v1+ engine: RunEngine | None = Nonenavigation_goal: str | None = Noneselector: str | None = Noneai_fallback: AIFallbackMode = AIFallbackMode.FALLBACK@@ -1283,7 +1283,7 @@ class NavigationBlockYAML(BlockYAML):navigation_goal: strurl: str | None = Nonetitle: str = ""- engine: RunEngine = RunEngine.skyvern_v1+ engine: RunEngine | None = Noneerror_code_mapping: dict[str, str] | None = Nonemax_retries: int = 0max_steps_per_run: int | None = None@@ -1308,7 +1308,7 @@ class ExtractionBlockYAML(BlockYAML):data_extraction_goal: strurl: str | None = Nonetitle: str = ""- engine: RunEngine = RunEngine.skyvern_v1+ engine: RunEngine | None = Nonedata_schema: dict[str, Any] | list | str | None = Nonemax_retries: int = 0max_steps_per_run: int | None = None@@ -1328,7 +1328,7 @@ class LoginBlockYAML(BlockYAML):url: str | None = Nonetitle: str = ""- engine: RunEngine = RunEngine.skyvern_v1+ engine: RunEngine | None = Nonenavigation_goal: str | None = Noneerror_code_mapping: dict[str, str] | None = Nonemax_retries: int = 0@@ -1409,7 +1409,7 @@ class FileDownloadBlockYAML(BlockYAML):navigation_goal: strurl: str | None = Nonetitle: str = ""- engine: RunEngine = RunEngine.skyvern_v1+ engine: RunEngine | None = Noneerror_code_mapping: dict[str, str] | None = Nonemax_retries: int = 0max_steps_per_run: int | None = Nonediff --git a/skyvern/services/pdf_import_service.py b/skyvern/services/pdf_import_service.pyindex 7077bab77..9c03fe463 100644--- a/skyvern/services/pdf_import_service.py+++ b/skyvern/services/pdf_import_service.py@@ -144,10 +144,6 @@ class PDFImportService:if keys_to_include:blk["parameter_keys"] = keys_to_include- # Ensure engine where needed- if blk.get("block_type") in {"navigation", "action", "extraction", "login", "file_download"}:- blk.setdefault("engine", "skyvern-1.0")-# Ensure url exists (can be empty string)if blk.get("block_type") in {"navigation", "action", "extraction", "file_download"}:if blk.get("url") is None:diff --git a/skyvern/services/script_service.py b/skyvern/services/script_service.pyindex 53e0a61e5..caf032064 100644--- a/skyvern/services/script_service.py+++ b/skyvern/services/script_service.py@@ -1419,12 +1419,13 @@ async def _detect_user_defined_errors(return []-def _resolve_original_block_engine(cache_key: str, workflow: Workflow) -> RunEngine | None:+def _resolve_original_block_engine(cache_key: str, workflow: Workflow, workflow_run_id: str | None) -> RunEngine | None:# Recursive: a cached block inside a for/while loop must keep its engine too (labels are- # validated globally unique, so the first match is the block).+ # validated globally unique, so the first match is the block). Resolved rather than read, so an+ # unset engine follows the run's routing like the block itself would.for block in get_all_blocks(workflow.workflow_definition.blocks):if block.label == cache_key:- return block.engine if isinstance(block, BaseTaskBlock) else None+ return block.resolve_engine(workflow_run_id) if isinstance(block, BaseTaskBlock) else Nonereturn None@@ -1672,7 +1673,7 @@ async def _fallback_to_ai_run(# Inherit the original block's engine when the caller left it at default; fail open to v1 on any miss.if engine == RunEngine.skyvern_v1:try:- resolved_engine = _resolve_original_block_engine(cache_key, workflow)+ resolved_engine = _resolve_original_block_engine(cache_key, workflow, workflow_run_id)if resolved_engine is not None and resolved_engine != RunEngine.skyvern_v1:engine = resolved_engineLOG.debug(diff --git a/tests/unit/_workflow_block_engine_fakes.py b/tests/unit/_workflow_block_engine_fakes.pyindex 6db4b8447..1e3b33588 100644--- a/tests/unit/_workflow_block_engine_fakes.py+++ b/tests/unit/_workflow_block_engine_fakes.py@@ -118,6 +118,7 @@ async def resolve_arm(first_version_error: Exception | None = None,workflow_status: WorkflowStatus = WorkflowStatus.published,trigger_type: WorkflowRunTriggerType | None = WorkflowRunTriggerType.api,+ takes_default_engine: bool | None = True,) -> Resolution:async def read_birth_timestamp(workflow_permanent_id: str, organization_id: str) -> datetime | None:if first_version_error is not None:@@ -161,6 +162,7 @@ async def resolve_arm(workflow_status=workflow_status,trigger_type=trigger_type,ineligibility_reason=ineligibility_reason,+ takes_default_engine=takes_default_engine,)logged = dict(mock_log.info.call_args.kwargs) if mock_log.info.call_args else {}return Resolution(log=logged, birth_reads=birth_reads, warnings=list(mock_log.warning.call_args_list))diff --git a/tests/unit/test_config.py b/tests/unit/test_config.pyindex ad8b5a94e..3bd993356 100644--- a/tests/unit/test_config.py+++ b/tests/unit/test_config.py@@ -133,9 +133,10 @@ def test_api_limit_concurrency_rejects_negative(monkeypatch: pytest.MonkeyPatch)("disabled", None),],)[email protected]("setting", ["TASK_V3_DEFAULT_ENGINE_WORKFLOW_CUTOFF", "TASK_V3_CHOSEN_ENGINE_CUTOFF"])def test_task_v3_default_engine_workflow_cutoff_env_values(- monkeypatch: pytest.MonkeyPatch, raw_value: str, expected: datetime | None+ monkeypatch: pytest.MonkeyPatch, setting: str, raw_value: str, expected: datetime | None) -> None:- monkeypatch.setenv("TASK_V3_DEFAULT_ENGINE_WORKFLOW_CUTOFF", raw_value)+ monkeypatch.setenv(setting, raw_value)- assert Settings(_env_file=None).TASK_V3_DEFAULT_ENGINE_WORKFLOW_CUTOFF == expected+ assert getattr(Settings(_env_file=None), setting) == expecteddiff --git a/tests/unit/test_script_service_fallback_engine.py b/tests/unit/test_script_service_fallback_engine.pyindex 19410063b..467a6e863 100644--- a/tests/unit/test_script_service_fallback_engine.py+++ b/tests/unit/test_script_service_fallback_engine.py@@ -13,9 +13,10 @@ from unittest.mock import AsyncMock, MagicMock, patchimport pytest+from skyvern.config import settingsfrom skyvern.forge.sdk.core.skyvern_context import SkyvernContextfrom skyvern.forge.sdk.schemas.tasks import TaskStatus-from skyvern.forge.sdk.workflow.models.block import TaskBlock+from skyvern.forge.sdk.workflow.models.block import TaskBlock, takes_default_engine, v3_ab_ineligibility_reasonfrom skyvern.forge.sdk.workflow.models.parameter import OutputParameter, ParameterTypefrom skyvern.forge.sdk.workflow.models.workflow import Workflow, WorkflowDefinitionfrom skyvern.schemas.runs import RunEngine@@ -23,6 +24,7 @@ from skyvern.schemas.workflows import BlockTypefrom skyvern.services import script_servicefrom skyvern.webeye.actions.action_types import ActionTypefrom skyvern.webeye.actions.actions import Action, ActionStatus+from tests.unit._workflow_block_engine_fakes import FakeExperimentationProvider, resolve_armMODULE = "skyvern.services.script_service"@@ -39,7 +41,7 @@ def _make_output_parameter(key: str) -> OutputParameter:)-def _make_task_block(label: str, engine: RunEngine = RunEngine.skyvern_v1) -> TaskBlock:+def _make_task_block(label: str, engine: RunEngine | None = RunEngine.skyvern_v1) -> TaskBlock:return TaskBlock(label=label,output_parameter=_make_output_parameter(f"{label}_output"),@@ -177,6 +179,42 @@ async def test_fallback_respects_explicit_engine_without_lookup() -> None:assert _fallback_task_block(app).engine == RunEngine.skyvern_v2+ ("born_at", "expected"),+ [+ (datetime(2026, 10, 2, tzinfo=timezone.utc), RunEngine.skyvern_v3),+ (datetime(2026, 9, 1, tzinfo=timezone.utc), RunEngine.skyvern_v1),+ ],+)+async def test_fallback_of_an_unset_engine_block_follows_the_chosen_engine_cutoff(+ monkeypatch: pytest.MonkeyPatch, born_at: datetime, expected: RunEngine+) -> None:+ # A script run of a workflow born past the cutoff: its cached block's AI fallback must run where+ # the same block would run uncached, v3, not on the skyvern_v1 an unset engine used to mean.+ monkeypatch.setattr(settings, "TASK_V3_DEFAULT_ENGINE_WORKFLOW_CUTOFF", None)+ monkeypatch.setattr(settings, "TASK_V3_CHOSEN_ENGINE_CUTOFF", datetime(2026, 10, 1, tzinfo=timezone.utc))+ blocks = [_make_task_block("my_block", engine=None)]+ workflow = _make_workflow(blocks)+ context = _make_context()+ await resolve_arm(+ context,+ FakeExperimentationProvider(),+ workflow_run_id="wr_test",+ ineligibility_reason=v3_ab_ineligibility_reason(blocks, is_script_run=True),+ takes_default_engine=takes_default_engine(blocks),+ first_version_created_at=born_at,+ )+ app = _make_app(workflow)++ with patch(f"{MODULE}.app", app), patch(f"{MODULE}.skyvern_context.current", return_value=context):+ await script_service._fallback_to_ai_run(+ block_type=BlockType.NAVIGATION, cache_key="my_block", prompt="do the thing"+ )++ assert _fallback_task_block(app).engine == expected++def test_resolver_finds_loop_nested_block_engine() -> None:# A cached block inside a for-loop must keep its configured engine on fallback; the lookup# is recursive (labels are globally unique, nested included).@@ -191,8 +229,8 @@ def test_resolver_finds_loop_nested_block_engine() -> None:output_parameter=nested.output_parameter,)workflow = _make_workflow([loop])- assert script_service._resolve_original_block_engine("inner_block", workflow) == RunEngine.skyvern_v3- assert script_service._resolve_original_block_engine("missing", workflow) is None+ assert script_service._resolve_original_block_engine("inner_block", workflow, None) == RunEngine.skyvern_v3+ assert script_service._resolve_original_block_engine("missing", workflow, None) is Nonedef _make_run_context(values: dict[str, object]) -> MagicMock:diff --git a/tests/unit/test_validation_without_page_information.py b/tests/unit/test_validation_without_page_information.pyindex f89ce8aee..fdabafdaa 100644--- a/tests/unit/test_validation_without_page_information.py+++ b/tests/unit/test_validation_without_page_information.py@@ -85,7 +85,8 @@ def test_converter_defaults_engine_and_max_steps_per_run_when_omitted() -> None:historical hardcoded step cap of 2 (not the YAML field's own None default)."""block_yaml = ValidationBlockYAML(label="v", complete_criterion="x")block = block_yaml_to_block(block_yaml, {"v_output": _output_parameter("v")})- assert block.engine == RunEngine.skyvern_v1+ assert block.engine is None+ assert block.resolve_engine(None) == RunEngine.skyvern_v1assert block.max_steps_per_run == 2diff --git a/tests/unit/test_workflow_block_engine.py b/tests/unit/test_workflow_block_engine.pyindex 9207ad8c3..837a54f03 100644--- a/tests/unit/test_workflow_block_engine.py+++ b/tests/unit/test_workflow_block_engine.py@@ -13,6 +13,7 @@ from typing import Anyfrom unittest.mock import AsyncMock, MagicMock, patchimport pytest+from pydantic import BaseModel, ConfigDict, ValidationErrorfrom skyvern.config import settingsfrom skyvern.forge import app@@ -26,6 +27,7 @@ from skyvern.forge.sdk.experimentation.workflow_block_engine import (WORKFLOW_TASK_V3_AB_FLAG,WorkflowBlockEngineRouteReason,_as_utc,+ effective_default_engine,resolve_workflow_block_engine_arm,workflow_block_engine_override,)@@ -52,12 +54,14 @@ from skyvern.forge.sdk.workflow.models.block import (_evaluate_prompt_branch_conditions_batch,get_all_blocks,run_is_eligible_for_v3_ab,+ takes_default_engine,v3_ab_ineligibility_reason,)+from skyvern.forge.sdk.workflow.models.workflow import WorkflowDefinitionfrom skyvern.forge.sdk.workflow.service import WorkflowServicefrom skyvern.forge.taskv3.goal_composition import render_block_contextfrom skyvern.schemas.run_enums import RunEngine-from skyvern.schemas.workflows import BlockResult, BlockType, WorkflowStatus+from skyvern.schemas.workflows import BlockResult, BlockType, NavigationBlockYAML, WorkflowStatusfrom skyvern.services import script_servicefrom tests.unit._workflow_block_engine_fakes import (WORKFLOW_BLOCK_ENGINE_APP_TARGET,@@ -92,6 +96,7 @@ def _no_ambient_v3_default_cutoff(monkeypatch: pytest.MonkeyPatch) -> None:# Settings reads .env at import: without pinning it, every test here answers to whatever the# machine happens to export.monkeypatch.setattr(settings, "TASK_V3_DEFAULT_ENGINE_WORKFLOW_CUTOFF", None)+ monkeypatch.setattr(settings, "TASK_V3_CHOSEN_ENGINE_CUTOFF", None)@pytest.fixture@@ -375,6 +380,7 @@ async def test_noop_provider_never_queried_and_leaves_engine_unchanged(scoped_coworkflow_status=WorkflowStatus.published,trigger_type=WorkflowRunTriggerType.api,ineligibility_reason=None,+ takes_default_engine=True,)spy.assert_not_called()@@ -804,6 +810,7 @@ async def test_resolver_logs_the_ineligibility_reason(scoped_context: SkyvernConworkflow_status=WorkflowStatus.published,trigger_type=WorkflowRunTriggerType.api,ineligibility_reason=V3AbIneligibleReason.pinned_engine,+ takes_default_engine=True,)assert scoped_context.workflow_block_engine_override is None@@ -1010,6 +1017,7 @@ async def test_arm_offers_the_billing_tier_to_the_ab_flag_but_not_the_kill_switcworkflow_status=WorkflowStatus.published,trigger_type=WorkflowRunTriggerType.api,ineligibility_reason=None,+ takes_default_engine=True,)by_flag = {flag: properties or {} for flag, _distinct_id, properties in provider.calls}@@ -1042,6 +1050,7 @@ async def test_a_run_that_never_reaches_the_ab_pays_for_no_tier_lookup(scoped_coworkflow_status=WorkflowStatus.published,trigger_type=WorkflowRunTriggerType.api,ineligibility_reason=reason,+ takes_default_engine=True,)tier.assert_not_awaited()assert mock_log.info.call_args.kwargs["billing_tier"] is None@@ -1065,6 +1074,7 @@ async def test_a_failing_billing_tier_lookup_cannot_decide_the_arm(scoped_contexworkflow_status=WorkflowStatus.published,trigger_type=WorkflowRunTriggerType.api,ineligibility_reason=None,+ takes_default_engine=True,)assert scoped_context.workflow_block_engine_override == RunEngine.skyvern_v3@@ -1395,3 +1405,235 @@ async def test_a_missing_first_version_falls_back_to_the_flag_path(assert scoped_context.workflow_block_engine_override is Noneassert resolution.log["route_reason"] == WorkflowBlockEngineRouteReason.flag_bucket_control+++CHOSEN_ENGINE_CUTOFF = datetime(2026, 10, 1, tzinfo=UTC)+++async def _resolve_chosen_engine_run(+ context: SkyvernContext,+ monkeypatch: pytest.MonkeyPatch,+ blocks: list[BaseTaskBlock],+ *,+ run_id: str,+ cutoff: datetime | None,+ born_at: datetime,+ flags: dict[str, bool] | None = None,+ workflow_status: WorkflowStatus = WorkflowStatus.published,+) -> tuple[FakeExperimentationProvider, Any]:+ # Eligibility and the unset-engine fact are computed from the blocks by the same helpers the+ # workflow service calls, so a block-level regression in either reaches these assertions.+ monkeypatch.setattr(settings, "TASK_V3_CHOSEN_ENGINE_CUTOFF", cutoff)+ provider = FakeExperimentationProvider(flags if flags is not None else {WORKFLOW_TASK_V3_AB_FLAG: True})+ resolution = await resolve_arm(+ context,+ provider,+ workflow_run_id=run_id,+ ineligibility_reason=v3_ab_ineligibility_reason(blocks, is_script_run=False),+ takes_default_engine=takes_default_engine(blocks),+ billing_tier=BillingTier.ENTERPRISE,+ first_version_created_at=born_at,+ workflow_status=workflow_status,+ )+ return provider, resolution+++_AFTER = CHOSEN_ENGINE_CUTOFF.replace(tzinfo=None) + timedelta(seconds=1)+_BEFORE = CHOSEN_ENGINE_CUTOFF.replace(tzinfo=None) - timedelta(days=30)+_R = WorkflowBlockEngineRouteReason+++ ("cutoff", "born_at", "chosen", "expected_engine", "expected_reason"),+ [+ # Cutoff unset or workflow born before it: today's routing. The A/B is at 100% treatment, so an+ # unset or skyvern_v1 block is rerouted and a v2/v3 pin makes the run ineligible.+ (None, _AFTER, None, RunEngine.skyvern_v3, _R.flag_bucket_treatment),+ (None, _AFTER, RunEngine.skyvern_v1, RunEngine.skyvern_v3, _R.flag_bucket_treatment),+ (None, _AFTER, RunEngine.skyvern_v2, RunEngine.skyvern_v2, _R.ineligible),+ (None, _AFTER, RunEngine.skyvern_v3, RunEngine.skyvern_v3, _R.ineligible),+ (CHOSEN_ENGINE_CUTOFF, _BEFORE, None, RunEngine.skyvern_v3, _R.flag_bucket_treatment),+ (CHOSEN_ENGINE_CUTOFF, _BEFORE, RunEngine.skyvern_v1, RunEngine.skyvern_v3, _R.flag_bucket_treatment),+ (CHOSEN_ENGINE_CUTOFF, _BEFORE, RunEngine.skyvern_v2, RunEngine.skyvern_v2, _R.ineligible),+ (CHOSEN_ENGINE_CUTOFF, _BEFORE, RunEngine.skyvern_v3, RunEngine.skyvern_v3, _R.ineligible),+ # Born at or after the cutoff: an unset engine runs on v3 and a chosen one runs as written.+ (CHOSEN_ENGINE_CUTOFF, _AFTER, None, RunEngine.skyvern_v3, _R.new_workflow_v3_default),+ (CHOSEN_ENGINE_CUTOFF, _AFTER, RunEngine.skyvern_v1, RunEngine.skyvern_v1, _R.chosen_engine),+ (CHOSEN_ENGINE_CUTOFF, _AFTER, RunEngine.skyvern_v2, RunEngine.skyvern_v2, _R.chosen_engine),+ (CHOSEN_ENGINE_CUTOFF, _AFTER, RunEngine.skyvern_v3, RunEngine.skyvern_v3, _R.chosen_engine),+ ],+)+async def test_chosen_engine_cutoff_matrix(+ scoped_context: SkyvernContext,+ monkeypatch: pytest.MonkeyPatch,+ cutoff: datetime | None,+ born_at: datetime,+ chosen: RunEngine | None,+ expected_engine: RunEngine,+ expected_reason: WorkflowBlockEngineRouteReason,+) -> None:+ block = _make_block(TaskBlock, label="only", engine=chosen)+ provider, resolution = await _resolve_chosen_engine_run(+ scoped_context, monkeypatch, [block], run_id="wr_matrix", cutoff=cutoff, born_at=born_at+ )++ assert block.resolve_engine("wr_matrix") == expected_engine+ assert resolution.log["route_reason"] == expected_reason+ if expected_reason in (_R.new_workflow_v3_default, _R.chosen_engine):+ # Every tier, with no A/B draw and no billing-tier lookup.+ assert consulted_flags(provider) == [DISABLE_TASK_V3_FLAG]+ assert resolution.log["billing_tier"] is None+++async def test_a_post_cutoff_run_honors_each_block_on_its_own(+ scoped_context: SkyvernContext, monkeypatch: pytest.MonkeyPatch+) -> None:+ # The A/B keeps a run single-engine; a chosen-engine run does not, so a v3 pin (which makes the run+ # A/B-ineligible) must not hold the unset blocks on v1, and a block v3 cannot run stays on v1.+ unset = _make_block(NavigationBlock, label="unset", navigation_goal="Apply to the job")+ chose_v1 = _make_block(TaskBlock, label="chose_v1", engine=RunEngine.skyvern_v1)+ chose_v3 = _make_block(TaskBlock, label="chose_v3", engine=RunEngine.skyvern_v3)+ unsupported = _make_block(ValidationBlock, label="unsupported", complete_on_download=True)+ blocks: list[BaseTaskBlock] = [unset, chose_v1, chose_v3, unsupported]+ _provider, resolution = await _resolve_chosen_engine_run(+ scoped_context, monkeypatch, blocks, run_id="wr_mixed", cutoff=CHOSEN_ENGINE_CUTOFF, born_at=_AFTER+ )++ assert [block.resolve_engine("wr_mixed") for block in blocks] == [+ RunEngine.skyvern_v3,+ RunEngine.skyvern_v1,+ RunEngine.skyvern_v3,+ RunEngine.skyvern_v1,+ ]+ assert resolution.log["route_reason"] == _R.new_workflow_v3_default++[email protected]("chosen", [None, RunEngine.skyvern_v3])+async def test_kill_switch_wins_over_the_chosen_engine_cutoff(+ scoped_context: SkyvernContext, monkeypatch: pytest.MonkeyPatch, chosen: RunEngine | None+) -> None:+ block = _make_block(TaskBlock, label="only", engine=chosen)+ _provider, resolution = await _resolve_chosen_engine_run(+ scoped_context,+ monkeypatch,+ [block],+ run_id="wr_killed",+ cutoff=CHOSEN_ENGINE_CUTOFF,+ born_at=_AFTER,+ flags={WORKFLOW_TASK_V3_AB_FLAG: True, DISABLE_TASK_V3_FLAG: True},+ )++ assert scoped_context.workflow_block_engine_override is None+ resolution.birth_reads.assert_not_awaited()+ if chosen is None:+ assert block.resolve_engine("wr_killed") == RunEngine.skyvern_v1+ assert resolution.log["route_reason"] == _R.disabled+ else:+ # The pin is recorded as written and the dispatch gate stops it; its label predates this rule.+ assert block.resolve_engine("wr_killed") == RunEngine.skyvern_v3+ assert resolution.log["route_reason"] == _R.ineligible+++async def test_a_per_call_workflow_born_after_the_chosen_engine_cutoff_keeps_ab_routing(+ scoped_context: SkyvernContext, monkeypatch: pytest.MonkeyPatch+) -> None:+ # Every per-call platform workflow is born after any cutoff; enrolling them would move standing+ # API traffic, so their explicit skyvern_v1 stays routable.+ block = _make_block(TaskBlock, label="only", engine=RunEngine.skyvern_v1)+ _provider, resolution = await _resolve_chosen_engine_run(+ scoped_context,+ monkeypatch,+ [block],+ run_id="wr_per_call",+ cutoff=CHOSEN_ENGINE_CUTOFF,+ born_at=_AFTER,+ workflow_status=WorkflowStatus.auto_generated,+ )++ assert block.resolve_engine("wr_per_call") == RunEngine.skyvern_v3+ assert resolution.log["route_reason"] == _R.flag_bucket_treatment+ resolution.birth_reads.assert_not_awaited()+++def test_an_omitted_engine_is_unset_and_a_stored_one_is_kept() -> None:+ omitted = NavigationBlockYAML(label="nav", url="https://example.com", navigation_goal="Go")+ stored = NavigationBlockYAML.model_validate(+ {"label": "nav", "url": "https://example.com", "navigation_goal": "Go", "engine": "skyvern-1.0"}+ )++ assert omitted.engine is None+ assert stored.engine == RunEngine.skyvern_v1+ block = _make_block(NavigationBlock, label="nav", navigation_goal="Go")+ assert block.engine is None+ assert type(block).model_validate(block.model_dump()).engine is None+ # Unset routes like skyvern_v1 before the cutoff, so it never reads as a pin.+ assert v3_ab_ineligibility_reason([block], is_script_run=False) is None+ assert takes_default_engine([block]) is True+++def test_an_unset_engine_is_left_out_of_the_stored_definition_so_an_older_image_reads_it() -> None:+ # The field as the image before SKY-17436 declares it: a stored null fails validation there, so a+ # pod still on it during a rollout, or after a revert, could not load the workflow.+ class OlderImageTaskBlock(BaseModel):+ model_config = ConfigDict(extra="allow")+ engine: RunEngine = RunEngine.skyvern_v1++ unset = _make_block(NavigationBlock, label="unset", navigation_goal="Go")+ chosen = _make_block(TaskBlock, label="chosen", engine=RunEngine.skyvern_v3)+ stored = WorkflowDefinition(parameters=[], blocks=[unset, chosen]).model_dump(mode="json")++ assert "engine" not in stored["blocks"][0]+ assert [OlderImageTaskBlock.model_validate(b).engine for b in stored["blocks"]] == [+ RunEngine.skyvern_v1,+ RunEngine.skyvern_v3,+ ]+ reloaded = WorkflowDefinition.model_validate(stored)+ assert [b.engine for b in reloaded.blocks] == [None, RunEngine.skyvern_v3]+ with pytest.raises(ValidationError):+ OlderImageTaskBlock.model_validate({"engine": None})+++async def test_both_cutoff_rules_share_one_birth_read(+ scoped_context: SkyvernContext, monkeypatch: pytest.MonkeyPatch, v3_default_cutoff: datetime+) -> None:+ # Born between the two cutoffs: the chosen-engine rule reads the birth and declines, then the+ # self-serve rule needs the same timestamp.+ monkeypatch.setattr(settings, "TASK_V3_CHOSEN_ENGINE_CUTOFF", CHOSEN_ENGINE_CUTOFF)+ resolution = await resolve_arm(+ scoped_context,+ FakeExperimentationProvider(),+ workflow_run_id="wr_between",+ ineligibility_reason=None,+ billing_tier=BillingTier.SELF_SERVE,+ first_version_created_at=CHOSEN_ENGINE_CUTOFF - timedelta(days=1),+ )++ assert resolution.log["route_reason"] == _R.new_self_serve_workflow_default+ assert resolution.birth_reads.await_count == 1+++ ("cutoff", "born_at", "expected"),+ [+ (None, _AFTER, None),+ (CHOSEN_ENGINE_CUTOFF, _BEFORE, None),+ (CHOSEN_ENGINE_CUTOFF, _AFTER, RunEngine.skyvern_v3),+ ],+)+async def test_effective_default_engine_follows_the_chosen_engine_cutoff(+ monkeypatch: pytest.MonkeyPatch, cutoff: datetime | None, born_at: datetime, expected: RunEngine | None+) -> None:+ monkeypatch.setattr(settings, "TASK_V3_CHOSEN_ENGINE_CUTOFF", cutoff)+ with patch(WORKFLOW_BLOCK_ENGINE_APP_TARGET) as mock_app:+ mock_app.EXPERIMENTATION_PROVIDER = FakeExperimentationProvider()+ mock_app.DATABASE.workflows.get_workflow_permanent_id_created_at = AsyncMock(return_value=born_at)+ assert await effective_default_engine("wpid_1", "org_1") == expected+ # With no flag provider the resolver never applies the rule, so the editor must not claim it.+ mock_app.EXPERIMENTATION_PROVIDER = NoOpExperimentationProvider()+ assert await effective_default_engine("wpid_1", "org_1") is Nonediff --git a/tests/unit/workflow/test_cache_invalidation.py b/tests/unit/workflow/test_cache_invalidation.pyindex d6711b85b..ea078100f 100644--- a/tests/unit/workflow/test_cache_invalidation.py+++ b/tests/unit/workflow/test_cache_invalidation.py@@ -5,12 +5,19 @@ Verifies that changes to the model field (both at workflow settings level and bldo not trigger cache invalidation."""-from datetime import datetime, timezone+from datetime import datetime, timedelta, timezone+from unittest.mock import AsyncMock, MagicMock, patch+import pytest++from skyvern.config import settingsfrom skyvern.forge.sdk.workflow.models.block import BlockType, CodeBlock, CodeBlockStep, TaskBlockfrom skyvern.forge.sdk.workflow.models.parameter import OutputParameter, ParameterType-from skyvern.forge.sdk.workflow.models.workflow import WorkflowDefinition-from skyvern.forge.sdk.workflow.service import _get_workflow_definition_core_data+from skyvern.forge.sdk.workflow.models.workflow import Workflow, WorkflowDefinition+from skyvern.forge.sdk.workflow.service import WorkflowService, _get_workflow_definition_core_data+from skyvern.forge.sdk.workflow.workflow_definition_converter import convert_workflow_definition+from skyvern.schemas.run_enums import RunEngine+from skyvern.schemas.workflows import WorkflowDefinitionYAMLfrom skyvern.webeye.actions.action_types import ActionType@@ -215,3 +222,104 @@ class TestCacheInvalidation:# These should be identical (timestamps and IDs are excluded)assert core_data1 == core_data2, "Timestamps and IDs should be excluded from comparison"+++_TASK_BLOCK_LABELS = ("open", "approve", "nav")+_CUTOFF = datetime(2026, 10, 1, tzinfo=timezone.utc)+++def _yaml_definition(nav_engine: str | None) -> WorkflowDefinitionYAML:+ nav: dict = {"block_type": "navigation", "label": "nav", "url": "https://example.com", "navigation_goal": "Go"}+ if nav_engine is not None:+ nav["engine"] = nav_engine+ return WorkflowDefinitionYAML.model_validate(+ {+ "parameters": [],+ "blocks": [+ {"block_type": "goto_url", "label": "open", "url": "https://example.com"},+ {"block_type": "human_interaction", "label": "approve", "recipients": ["[email protected]"]},+ nav,+ ],+ }+ )+++def _as_the_previous_image_stored_it(definition: WorkflowDefinition) -> dict:+ # Before SKY-17436 every task block's engine defaulted to skyvern-1.0 and was always written.+ stored = definition.model_dump(mode="json")+ for block in stored["blocks"]:+ if block["label"] in _TASK_BLOCK_LABELS and block.get("engine") is None:+ block["engine"] = RunEngine.skyvern_v1.value+ return stored+++async def _resave(+ stored: dict, resaved: WorkflowDefinition, monkeypatch: pytest.MonkeyPatch, *, born_at: datetime+) -> AsyncMock:+ """Save ``resaved`` over a workflow whose previous version holds ``stored``; returns the cache clear."""+ monkeypatch.setattr(settings, "TASK_V3_CHOSEN_ENGINE_CUTOFF", _CUTOFF)+ now = datetime.now(timezone.utc)+ previous = Workflow(+ workflow_id="w_1",+ organization_id="o_1",+ title="t",+ workflow_permanent_id="wpid_1",+ version=1,+ is_saved_task=False,+ workflow_definition=WorkflowDefinition.model_validate(stored),+ created_at=now,+ modified_at=now,+ )+ service = WorkflowService()+ database = MagicMock()+ database.workflows.get_workflow_by_permanent_id = AsyncMock(return_value=previous)+ with (+ patch("skyvern.forge.sdk.workflow.service.app") as mock_app,+ patch("skyvern.forge.sdk.experimentation.workflow_block_engine.app") as engine_app,+ patch.object(service, "_partition_cached_blocks", AsyncMock(return_value=([MagicMock()], []))),+ patch.object(service, "_clear_cached_block_groups", AsyncMock()) as clear_groups,+ ):+ mock_app.DATABASE = database+ engine_app.DATABASE.workflows.get_workflow_permanent_id_created_at = AsyncMock(return_value=born_at)+ await service.maybe_delete_cached_code(+ previous.model_copy(update={"version": 2}), workflow_definition=resaved, organization_id="o_1"+ )+ return clear_groups+++ "nav_engine",+ [+ # The editor, which sends the stored skyvern-1.0 back.+ RunEngine.skyvern_v1.value,+ # An API caller re-sending the YAML it created the workflow with, which omitted the engine.+ None,+ ],+)+async def test_an_unchanged_resave_of_a_previously_stored_definition_keeps_its_cached_scripts(+ monkeypatch: pytest.MonkeyPatch, nav_engine: str | None+) -> None:+ resaved = convert_workflow_definition(_yaml_definition(nav_engine), "w_1")+ stored = _as_the_previous_image_stored_it(resaved)++ clear_groups = await _resave(stored, resaved, monkeypatch, born_at=_CUTOFF - timedelta(days=30))++ clear_groups.assert_not_awaited()+ if nav_engine is not None:+ # What the editor saves is what was stored, in every engine field.+ assert resaved.model_dump(mode="json") == stored+++async def test_past_the_cutoff_switching_a_legacy_pin_to_default_is_a_change(monkeypatch: pytest.MonkeyPatch) -> None:+ # Unset runs on v3 there and skyvern-1.0 is an explicit pin, so the cached script must be cleared;+ # the engine-inert blocks around it still compare equal.+ stored = convert_workflow_definition(_yaml_definition(RunEngine.skyvern_v1.value), "w_1").model_dump(mode="json")+ resaved = convert_workflow_definition(_yaml_definition(None), "w_1")+ born_after = _CUTOFF + timedelta(days=1)++ clear_groups = await _resave(stored, resaved, monkeypatch, born_at=born_after)++ clear_groups.assert_awaited_once()+ assert clear_groups.await_args.kwargs["plan"].label == "nav"diff --git a/tests/unit/workflow/test_validation_action_yaml_field_passthrough.py b/tests/unit/workflow/test_validation_action_yaml_field_passthrough.pyindex 60dd2b6cc..f68b3ff7e 100644--- a/tests/unit/workflow/test_validation_action_yaml_field_passthrough.py+++ b/tests/unit/workflow/test_validation_action_yaml_field_passthrough.py@@ -80,7 +80,9 @@ def test_yaml_import_defaults_when_new_keys_omitted() -> None:validation_block = blocks["v"]assert isinstance(validation_block, ValidationBlock)- assert validation_block.engine == RunEngine.skyvern_v1+ # Stored unset, and resolved exactly like the old skyvern_v1 default outside a chosen-engine run.+ assert validation_block.engine is None+ assert validation_block.resolve_engine(None) == RunEngine.skyvern_v1assert validation_block.max_steps_per_run == 2action_block = blocks["a"]