Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
Run a new non-enterprise workflow's task blocks on Task V3 by default (SKY-16293) (#8545)
Shuchang Zhengfcc14e82776d
diff --git a/skyvern/config.py b/skyvern/config.pyindex 9df689538..80266bf4a 100644--- a/skyvern/config.py+++ b/skyvern/config.py@@ -1,6 +1,7 @@import loggingimport osimport platform+from datetime import datetimefrom enum import StrEnumfrom pathlib import Pathfrom typing import Any, Literal@@ -557,6 +558,20 @@ class Settings(BaseSettings):# executes only in the sandboxed runner and is withheld whenever that runner is unavailable --# there is no in-process execution path to fall back to.TASK_V3_CODE_TOOL_SURFACE: Literal["off", "add", "replace"] = "off"+ # Workflows whose permanent id was born at or after this instant run their task blocks on Task V3+ # when the organization resolves to the self-serve billing tier (an unknown tier is not enrolled),+ # bypassing WORKFLOW_TASK_V3_AB. None disables the rule (the OSS default). Setting it is not+ # enough on its own: the rule fires only for runs whose TASK_V3_NEW_WORKFLOW_DEFAULT_ROLLOUT+ # evaluation returns a conclusive true, so the cutoff stays inert until that flag enrols someone.+ # Every off-state of that flag -- disabled, deleted, 0%, an excluding condition, an evaluation+ # that raised, no flag provider at all -- leaves the run on the A/B, so there is no off-state that+ # enrols. A naive value is read as UTC. Per-call platform workflows are excluded twice over -- by+ # an auto_generated executing version, which covers the login, download_files, credential+ # test-login and SDK endpoints, and by a per-call trigger kind, which covers the job recipe+ # endpoints because those build a published definition. Enrolled runs are not randomized, so every+ # per-arm read must exclude them by route_reason and read them against the unenrolled control cell+ # instead.+ TASK_V3_DEFAULT_ENGINE_WORKFLOW_CUTOFF: datetime | None = None# VOLCENGINE (Doubao)ENABLE_VOLCENGINE: bool = False@@ -1012,6 +1027,16 @@ class Settings(BaseSettings):return Nonereturn value+ @field_validator("TASK_V3_DEFAULT_ENGINE_WORKFLOW_CUTOFF", mode="before")+ @classmethod+ def _task_v3_default_engine_workflow_cutoff_off_sentinels(cls, value: Any) -> Any:+ # This setting is the rule's settings-side kill path, and blanking an already-set env var is+ # how it gets turned off mid-incident. Without this, every off-spelling fails datetime+ # validation and crashes the process at import instead of disabling the rule.+ if value is None or str(value).strip().lower() in ("", "0", "none", "null", "off", "disabled"):+ return None+ return value+@field_validator("WORKER_STALL_DUMP_SECONDS", mode="before")@classmethoddef _worker_stall_dump_seconds_or_default(cls, value: Any) -> int:diff --git a/skyvern/forge/sdk/experimentation/workflow_block_engine.py b/skyvern/forge/sdk/experimentation/workflow_block_engine.pyindex ed8d202b9..bd96c0ce8 100644--- a/skyvern/forge/sdk/experimentation/workflow_block_engine.py+++ b/skyvern/forge/sdk/experimentation/workflow_block_engine.py@@ -1,14 +1,19 @@from __future__ import annotations+from datetime import UTC, datetime+from enum import StrEnumfrom typing import TYPE_CHECKINGimport structlog+from skyvern.config import settingsfrom skyvern.forge import appfrom skyvern.forge.sdk.core import skyvern_context+from skyvern.forge.sdk.db.enums import JOB_RECIPE_WORKFLOW_RUN_TRIGGER_TYPES, WorkflowRunTriggerTypefrom skyvern.forge.sdk.experimentation.billing_tier import BILLING_TIER_PROPERTY, BillingTier-from skyvern.forge.sdk.experimentation.providers import NoOpExperimentationProvider+from skyvern.forge.sdk.experimentation.providers import BaseExperimentationProvider, NoOpExperimentationProviderfrom skyvern.schemas.run_enums import RunEngine+from skyvern.schemas.workflows import WorkflowStatusif TYPE_CHECKING:# Runtime import would cycle: block.py imports workflow_block_engine_override from this module.@@ -18,6 +23,63 @@ LOG = structlog.get_logger()WORKFLOW_TASK_V3_AB_FLAG = "WORKFLOW_TASK_V3_AB"DISABLE_TASK_V3_FLAG = "DISABLE_TASK_V3"+TASK_V3_NEW_WORKFLOW_DEFAULT_ROLLOUT_FLAG = "TASK_V3_NEW_WORKFLOW_DEFAULT_ROLLOUT"++# Trigger kinds whose run executes a workflow the platform minted for that one request rather than+# one a customer built. The job-recipe endpoints are the whole set today: each call creates a fresh+# workflow_permanent_id, runs it immediately, and never runs it again, so a per-workflow rule cannot+# match one against a control run of the same workflow.+PER_CALL_WORKFLOW_RUN_TRIGGER_TYPES: frozenset[WorkflowRunTriggerType] = JOB_RECIPE_WORKFLOW_RUN_TRIGGER_TYPES+++class WorkflowBlockEngineRouteReason(StrEnum):+ """Why a workflow run landed on its arm, mirroring the bare-task RunRouteReason.++ Both arms persist the same engine on a control run, so without this the log cannot separate a+ bucketed control run from one that was never randomized -- and+ ``new_self_serve_workflow_default`` runs were never randomized, so every per-arm read has to+ exclude them by this field rather than by ``arm``. ``flag_bucket_control`` is therefore the+ randomized control cell and nothing else: a run the A/B flag never answered for is+ ``flag_undefined`` and a run whose evaluation raised is ``flag_error``, both on control.+ """++ flag_bucket_treatment = "flag_bucket_treatment"+ flag_bucket_control = "flag_bucket_control"+ new_self_serve_workflow_default = "new_self_serve_workflow_default"+ ineligible = "ineligible"+ disabled = "disabled"+ flag_undefined = "flag_undefined"+ flag_error = "flag_error"+++class NewWorkflowDefaultRollout(StrEnum):+ """How ``TASK_V3_NEW_WORKFLOW_DEFAULT_ROLLOUT`` answered for a run the v3 default would enrol.++ Only ``enrolled`` -- a conclusive True -- lets the rule fire. ``not_enrolled`` is a conclusive+ False: the flag is inactive, the run fell outside the percentage, or a condition excluded it.+ ``undefined`` means the key never resolved at all: nobody created it, or local evaluation has no+ snapshot row for it yet. ``error`` means the evaluation raised. The last three are logged apart+ from each other because only ``not_enrolled`` is a randomized cell -- a read that wants this+ rule's control must pin that value, not the boolean.+ """++ enrolled = "enrolled"+ not_enrolled = "not_enrolled"+ undefined = "undefined"+ error = "error"+++# Every resolution but a conclusive True leaves the run on the A/B path, so every off-state of the+# flag -- inactive, deleted, 0%, a condition that excluded the run, an evaluation that raised -- turns+# the rule off. An inactive flag is the one that has to be safe: posthog's local evaluator answers a+# conclusive False for it (``_compute_flag_locally``, "if not active: return False"), which is the+# gesture an operator reaches for mid-incident. Derived from the enum so a resolution added later+# leaves the run on the A/B instead of enrolling it.+_NOT_ENROLLED_RESOLUTIONS = frozenset(NewWorkflowDefaultRollout) - {NewWorkflowDefaultRollout.enrolled}+++def _rollout_enrols(rollout: NewWorkflowDefaultRollout | None) -> bool:+ return rollout is not None and rollout not in _NOT_ENROLLED_RESOLUTIONSasync def task_v3_disabled(distinct_id: str, organization_id: str | None) -> bool:@@ -45,12 +107,130 @@ async def _billing_tier_for_arm(organization_id: str | None) -> BillingTier:return BillingTier.UNKNOWN+def _as_utc(value: datetime) -> datetime:+ # workflows.created_at is written by datetime.utcnow(), i.e. naive UTC, while the cutoff setting+ # arrives aware or naive depending on how it was spelled in the environment.+ return value.replace(tzinfo=UTC) if value.tzinfo is None else value.astimezone(UTC)+++async def _workflow_is_new_for_v3_default(workflow_permanent_id: str | None, organization_id: str | None) -> bool:+ """Whether this permanent id was born at or after the v3-default cutoff.++ Never raises and never answers True on a read it could not complete: the rule can only add+ treatment, so any doubt has to leave the run on the arm the A/B would have given it.+ """+ cutoff = settings.TASK_V3_DEFAULT_ENGINE_WORKFLOW_CUTOFF+ if cutoff is None or not workflow_permanent_id or not organization_id:+ return False+ try:+ # The earliest version's timestamp, deleted versions included, is when the permanent id was+ # born. Reading the version this run executes would instead enrol every long-lived workflow+ # saved once after the cutoff.+ born_at = await app.DATABASE.workflows.get_workflow_permanent_id_created_at(+ workflow_permanent_id, organization_id+ )+ if born_at is None:+ return False+ # Inside the try with the read: a rule that raised here would reach the resolver's catch-all+ # and send the run to control, which is a different arm than the A/B would have given it.+ return _as_utc(born_at) >= _as_utc(cutoff)+ except Exception:+ LOG.warning(+ "Failed to read the workflow's birth timestamp for the Task V3 default rule",+ workflow_permanent_id=workflow_permanent_id,+ exc_info=True,+ )+ return False+++async def _ab_flag_puts_run_in_treatment(+ provider: BaseExperimentationProvider,+ *,+ workflow_run_id: str,+ organization_id: str | None,+ workflow_permanent_id: str | None,+ billing_tier: BillingTier,+) -> bool | None:+ """Whether the percentage knob buckets this run into treatment, or ``None`` if it never answered.++ ``resolve_feature_flag_strict`` rather than the boolean resolver so neither answer that is not a+ bucket collapses into one: both arms leave the same engine on a control run, so a swallowed error+ or an unresolved key would otherwise enter every per-arm read as a bucketed control run that was+ never randomized. A raised evaluation propagates to the resolver's catch-all and is labelled+ ``flag_error``; ``None`` -- the key is absent from the local snapshot, or carries a condition+ local evaluation cannot answer -- is returned as itself and labelled ``flag_undefined``. The arm+ is control for all three. Cached on the same (flag, distinct_id, properties) key the boolean+ resolver was, in the provider's own strict map, so this stays one evaluation per run.+ """+ return await provider.resolve_feature_flag_strict(+ WORKFLOW_TASK_V3_AB_FLAG,+ workflow_run_id,+ properties={+ "organization_id": organization_id,+ "workflow_permanent_id": workflow_permanent_id or "not_workflow",+ BILLING_TIER_PROPERTY: billing_tier.value,+ },+ )+++async def _resolve_new_workflow_default_rollout(+ provider: BaseExperimentationProvider,+ *,+ workflow_run_id: str,+ organization_id: str | None,+ workflow_permanent_id: str | None,+ billing_tier: BillingTier,+) -> NewWorkflowDefaultRollout:+ """How the rollout flag answered for a run the new-workflow v3 default would otherwise enrol.++ This flag is what enrols: the share it does not enrol continues down ``WORKFLOW_TASK_V3_AB``+ exactly as if the rule did not exist, which is where this population's concurrent control comes+ from. It is also the only lever that stops the rule for less than everybody -- the cutoff is a+ setting and needs a restart, and a condition on ``WORKFLOW_TASK_V3_AB`` cannot reach a run that+ never evaluates it. Turning it off de-enrols rather than forcing v1: an unenrolled run can still+ be treated by the A/B, so ``DISABLE_TASK_V3`` is the lever for getting an organization off v3.++ ``resolve_feature_flag_strict`` rather than the boolean resolver, so an evaluation that raised is+ labelled rather than collapsed: both PostHog providers swallow one into ``None`` and ``bool()`` it+ to ``False``, which would report an error as a real "outside the percentage".++ Only a conclusive ``True`` enrols. ``False``, ``None`` and a raised evaluation all leave the run on+ the A/B, so every off-gesture on this flag -- disable, delete, 0%, an excluding condition -- turns+ the rule off, and the cutoff does nothing until the flag resolves ``True`` for someone. A+ condition on it must be written on person properties: the arm-resolving processes evaluate+ PostHog locally, and a cohort, group or flag-dependency condition resolves to ``None``, which+ enrols nobody.+ """+ try:+ resolved = await provider.resolve_feature_flag_strict(+ TASK_V3_NEW_WORKFLOW_DEFAULT_ROLLOUT_FLAG,+ workflow_run_id,+ properties={+ "organization_id": organization_id,+ "workflow_permanent_id": workflow_permanent_id,+ BILLING_TIER_PROPERTY: billing_tier.value,+ },+ )+ except Exception:+ LOG.warning(+ "Failed to evaluate the new-workflow v3 default rollout; leaving this run on the A/B",+ workflow_run_id=workflow_run_id,+ exc_info=True,+ )+ return NewWorkflowDefaultRollout.error+ if resolved is None:+ return NewWorkflowDefaultRollout.undefined+ return NewWorkflowDefaultRollout.enrolled if resolved else NewWorkflowDefaultRollout.not_enrolled++async def resolve_workflow_block_engine_arm(context: skyvern_context.SkyvernContext,*,workflow_run_id: str,organization_id: str | None,workflow_permanent_id: str | None,+ workflow_status: WorkflowStatus,+ trigger_type: WorkflowRunTriggerType | None,ineligibility_reason: V3AbIneligibleReason | None,) -> None:"""Resolve the workflow-block engine A/B once at execution start and pin the arm on the context.@@ -61,6 +241,21 @@ async def resolve_workflow_block_engine_arm(own workflow_run_id and its own definition -- re-resolves instead of inheriting an arm that wasnever checked against its blocks. Control (no override) is the safe default and the outcome ofevery failure.++ A self-serve organization's workflow born at or after+ ``settings.TASK_V3_DEFAULT_ENGINE_WORKFLOW_CUTOFF`` takes v3 without the percentage being+ consulted, so those runs are marked ``new_self_serve_workflow_default`` for every per-arm read to+ exclude. ``TASK_V3_NEW_WORKFLOW_DEFAULT_ROLLOUT``, read only for the runs that rule would enrol,+ is what enrols them; the share it leaves alone continues down the A/B, which is where this+ population's concurrent control and its scoped kill come from. The rule fires only on a+ conclusive True, so the cutoff is inert until the flag resolves True for someone, and disabling,+ deleting or zeroing the flag turns the rule off.++ ``workflow_status`` is the status of the version this run executes and ``trigger_type`` is how+ the run was launched. Together they are what separates a workflow a customer keeps from the+ per-call throwaways several endpoints mint and run in one request: most of those run while still+ auto_generated, and the job-recipe endpoints build a published definition, so neither test alone+ covers them."""if context.workflow_block_engine_resolved_run_id == workflow_run_id:return@@ -76,30 +271,74 @@ async def resolve_workflow_block_engine_arm(override: RunEngine | None = Nonerun_is_eligible = ineligibility_reason is Nonebilling_tier: BillingTier | None = None+ route_reason = WorkflowBlockEngineRouteReason.ineligible+ rollout: NewWorkflowDefaultRollout | None = Nonetry:if run_is_eligible:# The kill switch, shared with the dispatch gate via task_v3_disabled so both- # evaluations use the same cache key, wins over the experiment. A kill flipped- # mid-run still takes effect at dispatch while these rows already read v3:- # stopping the run wins over its attribution. Read FIRST, and before the billing- # lookup below: a Redis or pooler incident is exactly when someone flips this, and- # this lock is held throughout, so the kill switch must not queue behind that call.- disabled = await task_v3_disabled(workflow_run_id, organization_id)- # billing_tier rides along so a release condition can hold enterprise and self-serve- # at different percentages. Deliberately not passed to task_v3_disabled: that flag's- # two callers must build identical provider cache keys, see its docstring.- if not disabled:+ # evaluations use the same cache key, wins over the experiment and over the+ # new-workflow default below. A kill flipped mid-run still takes effect at dispatch+ # while these rows already read v3: stopping the run wins over its attribution. Read+ # FIRST, and before the billing lookup below: a Redis or pooler incident is exactly+ # when someone flips this, and this lock is held throughout, so the kill switch must+ # not queue behind that call.+ if await task_v3_disabled(workflow_run_id, organization_id):+ route_reason = WorkflowBlockEngineRouteReason.disabled+ else:+ # billing_tier rides along so a release condition can hold enterprise and+ # self-serve at different percentages. Deliberately not passed to+ # task_v3_disabled: that flag's two callers must build identical provider cache+ # keys, see its docstring.billing_tier = await _billing_tier_for_arm(organization_id)- if await provider.is_feature_enabled_cached(- WORKFLOW_TASK_V3_AB_FLAG,- workflow_run_id,- properties={- "organization_id": organization_id,- "workflow_permanent_id": workflow_permanent_id or "not_workflow",- BILLING_TIER_PROPERTY: billing_tier.value,- },- ):+ # The tier, the status and the trigger are checked first, so anything that+ # cannot use the rule pays for no workflow read. An auto_generated running+ # version is a per-call workflow the login, download_files, credential+ # test-login and SDK endpoints mint and run in one request, and enrolling those+ # would move standing API traffic rather than the workflows a customer keeps;+ # the birth version's status would be the wrong test, because the prompt box+ # writes version 1 as auto_generated and the editor save that keeps it writes+ # the next as published. The job-recipe endpoints are per-call too but build a+ # published definition, so only their trigger kind excludes them.+ takes_new_workflow_default = (+ billing_tier == BillingTier.SELF_SERVE+ and workflow_status != WorkflowStatus.auto_generated+ and trigger_type not in PER_CALL_WORKFLOW_RUN_TRIGGER_TYPES+ and await _workflow_is_new_for_v3_default(workflow_permanent_id, organization_id)+ )+ if takes_new_workflow_default:+ # Read only for the runs the rule would enrol, so the A/B population is never+ # exposed to this flag and its bucketing is unchanged.+ rollout = await _resolve_new_workflow_default_rollout(+ provider,+ workflow_run_id=workflow_run_id,+ organization_id=organization_id,+ workflow_permanent_id=workflow_permanent_id,+ billing_tier=billing_tier,+ )+ if takes_new_workflow_default and _rollout_enrols(rollout):+ # A workflow born at or after the cutoff runs its task blocks on v3 by+ # default, so the percentage knob is never consulted: the unenrolled share the+ # A/B then routes to control is the only concurrent control these runs have.override = RunEngine.skyvern_v3+ route_reason = WorkflowBlockEngineRouteReason.new_self_serve_workflow_default+ else:+ in_treatment = await _ab_flag_puts_run_in_treatment(+ provider,+ workflow_run_id=workflow_run_id,+ organization_id=organization_id,+ workflow_permanent_id=workflow_permanent_id,+ billing_tier=billing_tier,+ )+ if in_treatment:+ override = RunEngine.skyvern_v3+ route_reason = WorkflowBlockEngineRouteReason.flag_bucket_treatment+ elif in_treatment is None:+ # The flag never answered, so this run is on control without having been+ # randomized. Labelled apart from the bucket so the control cell every+ # per-arm read builds out of flag_bucket_control stays a randomized one.+ route_reason = WorkflowBlockEngineRouteReason.flag_undefined+ else:+ route_reason = WorkflowBlockEngineRouteReason.flag_bucket_controlexcept Exception:LOG.warning("Failed to resolve the workflow-block engine arm; using control",@@ -107,6 +346,7 @@ async def resolve_workflow_block_engine_arm(exc_info=True,)override = None+ route_reason = WorkflowBlockEngineRouteReason.flag_errorcontext.workflow_block_engine_override = overridecontext.workflow_block_engine_resolved_run_id = workflow_run_idLOG.info(@@ -114,6 +354,18 @@ async def resolve_workflow_block_engine_arm(workflow_run_id=workflow_run_id,workflow_permanent_id=workflow_permanent_id,arm="treatment" if override else "control",+ route_reason=route_reason,+ # True only for a run this rule enrolled, which is every run it treated: the rule's+ # control cell is the unenrolled share the A/B then routed to control, so a read of the+ # rule compares route_reason=new_self_serve_workflow_default against+ # new_workflow_default_rollout_resolution=not_enrolled intersected with+ # flag_bucket_control -- an unenrolled run re-enters the A/B and can be treated there.+ new_workflow_default_rollout=_rollout_enrols(rollout),+ # None whenever the rollout flag was not consulted. The three non-enrolling resolutions+ # are indistinguishable in the boolean above, and only not_enrolled is a randomized cell,+ # so both a check of whether the flag is answering at all and the rule's control cell read+ # this field instead.+ new_workflow_default_rollout_resolution=rollout,run_is_eligible=run_is_eligible,ineligibility_reason=ineligibility_reason,# None whenever the A/B was never consulted -- an ineligible run, or one the kill switchdiff --git a/skyvern/forge/sdk/workflow/service.py b/skyvern/forge/sdk/workflow/service.pyindex e81c070e4..483a46406 100644--- a/skyvern/forge/sdk/workflow/service.py+++ b/skyvern/forge/sdk/workflow/service.py@@ -5827,6 +5827,8 @@ class WorkflowService:workflow_run_id=workflow_run_id,organization_id=organization_id,workflow_permanent_id=workflow_run.workflow_permanent_id,+ workflow_status=workflow.status,+ trigger_type=workflow_run.trigger_type,ineligibility_reason=v3_ab_ineligibility_reason(all_blocks, is_script_run=is_script_run),)else:diff --git a/tests/unit/test_config.py b/tests/unit/test_config.pyindex 49047c010..ad8b5a94e 100644--- a/tests/unit/test_config.py+++ b/tests/unit/test_config.py@@ -1,4 +1,5 @@import logging+from datetime import UTC, datetimeimport pytest@@ -113,3 +114,28 @@ def test_api_limit_concurrency_rejects_negative(monkeypatch: pytest.MonkeyPatch)with pytest.raises(ValueError):Settings(_env_file=None)+++ ("raw_value", "expected"),+ [+ ("2026-09-15T00:00:00Z", datetime(2026, 9, 15, tzinfo=UTC)),+ ("2026-09-15 00:00:00", datetime(2026, 9, 15)),+ ("2026-09-15", datetime(2026, 9, 15)),+ # This setting is the settings-side kill path for the new-workflow v3 default, and blanking+ # an already-set env var is how it gets turned off. A boot-time ValidationError would+ # CrashLoopBackOff every arm-resolving process during the incident the operator is ending.+ ("", None),+ ("0", None),+ ("none", None),+ ("null", None),+ ("off", None),+ ("disabled", None),+ ],+)+def test_task_v3_default_engine_workflow_cutoff_env_values(+ monkeypatch: pytest.MonkeyPatch, raw_value: str, expected: datetime | None+) -> None:+ monkeypatch.setenv("TASK_V3_DEFAULT_ENGINE_WORKFLOW_CUTOFF", raw_value)++ assert Settings(_env_file=None).TASK_V3_DEFAULT_ENGINE_WORKFLOW_CUTOFF == expecteddiff --git a/tests/unit/test_workflow_block_engine.py b/tests/unit/test_workflow_block_engine.pyindex d1df6a4aa..1580a177e 100644--- a/tests/unit/test_workflow_block_engine.py+++ b/tests/unit/test_workflow_block_engine.py@@ -5,22 +5,29 @@ workflow_run_blocks and the dispatched engine come from the same resolution.from __future__ import annotations+import timefrom collections.abc import Callable, Iterator-from datetime import UTC, datetime+from datetime import UTC, datetime, timedelta, timezonefrom types import SimpleNamespace-from typing import Any+from typing import Any, NamedTuplefrom unittest.mock import AsyncMock, MagicMock, patchimport pytest+from skyvern.config import settingsfrom skyvern.forge import appfrom skyvern.forge.sdk.core import skyvern_contextfrom skyvern.forge.sdk.core.skyvern_context import SkyvernContext+from skyvern.forge.sdk.db.enums import WorkflowRunTriggerTypefrom skyvern.forge.sdk.experimentation.billing_tier import BILLING_TIER_PROPERTY, BillingTierfrom skyvern.forge.sdk.experimentation.providers import BaseExperimentationProvider, NoOpExperimentationProviderfrom skyvern.forge.sdk.experimentation.workflow_block_engine import (DISABLE_TASK_V3_FLAG,+ TASK_V3_NEW_WORKFLOW_DEFAULT_ROLLOUT_FLAG,WORKFLOW_TASK_V3_AB_FLAG,+ NewWorkflowDefaultRollout,+ WorkflowBlockEngineRouteReason,+ _as_utc,resolve_workflow_block_engine_arm,workflow_block_engine_override,)@@ -51,7 +58,7 @@ from skyvern.forge.sdk.workflow.models.block import ()from skyvern.forge.sdk.workflow.service import WorkflowServicefrom skyvern.schemas.run_enums import RunEngine-from skyvern.schemas.workflows import BlockResult, BlockType+from skyvern.schemas.workflows import BlockResult, BlockType, WorkflowStatusfrom skyvern.services import script_servicefrom tests.unit.helpers import make_organizationfrom tests.unit.test_agent_task_v3 import (@@ -67,17 +74,70 @@ WORKFLOW_BLOCK_ENGINE_APP_TARGET = "skyvern.forge.sdk.experimentation.workflow_bclass _FakeExperimentationProvider(BaseExperimentationProvider):- def __init__(self, flags: dict[str, bool] | None = None, raise_error: bool = False) -> None:+ """Shaped like the two cloud providers, which is what makes a direction-sensitive flag testable.++ In production an evaluation error never escapes ``_is_feature_enabled``: both PostHog providers+ swallow it into ``None`` and ``bool()`` it to ``False``, so only ``_resolve_feature_flag_strict``+ can tell a failure from a real ``False``. A fake that raised from ``_is_feature_enabled`` would+ verify a shape no provider has, and would let a fail-safe that is inverted in production pass.++ ``False`` here is the answer an INACTIVE flag gives: posthog's local evaluator returns a+ conclusive ``False`` for one before it reads any filter, so "disable the flag" and "roll it to 0%"+ arrive as the same value, and a rule that fired on ``False`` would enrol everybody the moment an+ operator switched the flag off.+ """++ def __init__(+ self,+ flags: dict[str, bool] | None = None,+ raise_error: bool = False,+ strict_error_flags: set[str] | None = None,+ unresolvable_flags: set[str] | None = None,+ ) -> None:super().__init__()self.flags = dict(flags or {})+ # The rule fires only on a conclusive True, so a fake that did not mention this flag would+ # answer "undefined" and quietly take every enrolment test in this file off the rule and onto+ # the A/B path, where control and an unenrolled run look identical. Tests that want the other+ # resolutions say so, with a False here or with strict_error_flags/unresolvable_flags.+ self.flags.setdefault(TASK_V3_NEW_WORKFLOW_DEFAULT_ROLLOUT_FLAG, True)self.calls: list[tuple[str, str, dict | None]] = []self.raise_error = raise_error+ self.strict_error_flags = set(strict_error_flags or ())+ # A flag key local evaluation cannot answer: absent from the snapshot the poller wrote, or+ # carrying a condition only the PostHog API can resolve. Neither resolver raises; both+ # return None.+ self.unresolvable_flags = set(unresolvable_flags or ())++ async def _prepare_feature_flag_resolution(self, feature_name: str, *, cached: bool) -> None:+ # Where a provider does raise in production: the local provider reloads its flag snapshot+ # from the database here, and a database failure escapes instead of resolving to a value.+ if self.raise_error:+ raise RuntimeError("provider unavailable")- async def _is_feature_enabled(self, feature_name: str, distinct_id: str, properties: dict | None = None) -> bool:+ async def _evaluate(self, feature_name: str, distinct_id: str, properties: dict | None) -> bool | None:self.calls.append((feature_name, distinct_id, properties))- if self.raise_error:+ if feature_name in self.strict_error_flags:raise RuntimeError("provider unavailable")- return self.flags.get(feature_name, False)+ if feature_name in self.unresolvable_flags:+ return None+ return self.flags.get(feature_name)++ async def _resolve_feature_flag_strict(+ self, feature_name: str, distinct_id: str, properties: dict | None = None+ ) -> bool | None:+ return await self._evaluate(feature_name, distinct_id, properties)++ async def _resolve_feature_flag(+ self, feature_name: str, distinct_id: str, properties: dict | None = None+ ) -> bool | None:+ try:+ return await self._evaluate(feature_name, distinct_id, properties)+ except Exception:+ return None++ async def _is_feature_enabled(self, feature_name: str, distinct_id: str, properties: dict | None = None) -> bool:+ return bool(await self._resolve_feature_flag(feature_name, distinct_id, properties))async def _get_value(self, feature_name: str, distinct_id: str, properties: dict | None = None) -> str | None:return None@@ -96,6 +156,23 @@ def scoped_context() -> Iterator[SkyvernContext]:skyvern_context.reset()+class _Resolution(NamedTuple):+ log: dict[str, Any]+ birth_reads: AsyncMock+ warnings: list[Any]+++def _consulted_flags(provider: _FakeExperimentationProvider) -> list[str]:+ return [flag for flag, _distinct_id, _properties in provider.calls]+++# When the LATEST version of every workflow in this file was saved, deliberately after the+# v3_default_cutoff fixture below: that models an old workflow edited after the cutoff, so a resolver+# reading the version this run executes instead of the permanent id's birth flips the arm and is+# caught behaviorally rather than by an assertion about a mock's call kwargs.+_LATEST_VERSION_CREATED_AT = datetime(2026, 9, 20)++async def _resolve(context: SkyvernContext,provider: BaseExperimentationProvider,@@ -104,17 +181,75 @@ async def _resolve(ineligibility_reason: V3AbIneligibleReason | None,organization_id: str | None = "org_1",workflow_permanent_id: str | None = "wpid_1",-) -> None:- with patch(WORKFLOW_BLOCK_ENGINE_APP_TARGET) as mock_app:+ billing_tier: BillingTier = BillingTier.UNKNOWN,+ first_version_created_at: datetime | None = None,+ first_version_status: WorkflowStatus = WorkflowStatus.published,+ first_version_error: Exception | None = None,+ workflow_status: WorkflowStatus = WorkflowStatus.published,+ trigger_type: WorkflowRunTriggerType | None = WorkflowRunTriggerType.api,+) -> _Resolution:+ async def read_birth_timestamp(workflow_permanent_id: str, organization_id: str) -> datetime | None:+ if first_version_error is not None:+ raise first_version_error+ return first_version_created_at++ async def read_workflow_version(+ workflow_permanent_id: str, *, version: int | None = None, **_: Any+ ) -> SimpleNamespace | None:+ if version != 1:+ return SimpleNamespace(created_at=_LATEST_VERSION_CREATED_AT, status=WorkflowStatus.published)+ if first_version_created_at is None:+ return None+ return SimpleNamespace(created_at=first_version_created_at, status=first_version_status)++ birth_reads = AsyncMock(side_effect=read_birth_timestamp)+ with (+ patch(WORKFLOW_BLOCK_ENGINE_APP_TARGET) as mock_app,+ patch("skyvern.forge.sdk.experimentation.workflow_block_engine.LOG") as mock_log,+ ):mock_app.EXPERIMENTATION_PROVIDER = provider- stub_workflow_block_engine_app(mock_app)+ stub_workflow_block_engine_app(mock_app, billing_tier=billing_tier)+ # spec'd to the one repository the rule may reach, and to the two reads it could plausibly+ # use: on a bare MagicMock any attribute path answers, so a resolver reading a method the+ # real AgentDB does not have would pass here and raise in production, where the helper's+ # catch-all would bury it as "not a new workflow".+ database = MagicMock(spec=["workflows"])+ database.workflows = MagicMock(spec=["get_workflow_permanent_id_created_at", "get_workflow_by_permanent_id"])+ database.workflows.get_workflow_permanent_id_created_at = birth_reads+ # Answers whatever version is asked for, carrying the birth version's own status at version 1:+ # a resolver that regressed to deciding the per-call exclusion on the BIRTH version's status+ # reds the prompt-created case, and one that read the executing version's timestamp reds the+ # long-lived-workflow case, both behaviorally.+ database.workflows.get_workflow_by_permanent_id = AsyncMock(side_effect=read_workflow_version)+ mock_app.DATABASE = databaseawait resolve_workflow_block_engine_arm(context,workflow_run_id=workflow_run_id,organization_id=organization_id,workflow_permanent_id=workflow_permanent_id,+ workflow_status=workflow_status,+ trigger_type=trigger_type,ineligibility_reason=ineligibility_reason,)+ logged = dict(mock_log.info.call_args.kwargs) if mock_log.info.call_args else {}+ return _Resolution(log=logged, birth_reads=birth_reads, warnings=list(mock_log.warning.call_args_list))++[email protected](autouse=True)+def _no_ambient_v3_default_cutoff(monkeypatch: pytest.MonkeyPatch) -> None:+ # The rollout this file covers sets TASK_V3_DEFAULT_ENGINE_WORKFLOW_CUTOFF in the environment, and+ # Settings reads .env at import: without pinning it, every test here answers to whatever the+ # machine happens to export.+ monkeypatch.setattr(settings, "TASK_V3_DEFAULT_ENGINE_WORKFLOW_CUTOFF", None)+++def v3_default_cutoff(monkeypatch: pytest.MonkeyPatch, _no_ambient_v3_default_cutoff: None) -> datetime:+ # Ordered after the autouse pin above by requesting it, so the off-by-default pin cannot clobber+ # the cutoff a test asked for.+ cutoff = datetime(2026, 9, 15, tzinfo=UTC)+ monkeypatch.setattr(settings, "TASK_V3_DEFAULT_ENGINE_WORKFLOW_CUTOFF", cutoff)+ return cutoffdef test_mixed_eligibility_run_download_block_is_eligible(scoped_context: SkyvernContext) -> None:@@ -309,11 +444,48 @@ async def test_disable_flag_wins_over_ab_flag(scoped_context: SkyvernContext) ->@pytest.mark.asyncioasync def test_provider_exception_fails_closed_to_control(scoped_context: SkyvernContext) -> None:provider = _FakeExperimentationProvider(raise_error=True)- await _resolve(scoped_context, provider, workflow_run_id="wr_err", ineligibility_reason=None)+ resolution = await _resolve(scoped_context, provider, workflow_run_id="wr_err", ineligibility_reason=None)assert scoped_context.workflow_block_engine_override is Noneblock = _make_block(TaskBlock, label="err_block")assert block.resolve_engine("wr_err") == RunEngine.skyvern_v1+ # An unrandomized control run has to be separable from a bucketed one in the same read.+ assert resolution.log["route_reason"] == WorkflowBlockEngineRouteReason.flag_error+++async def test_a_failing_ab_evaluation_is_labelled_flag_error_not_bucketed_control(+ scoped_context: SkyvernContext,+) -> None:+ # A failed A/B evaluation and a real control bucket produce the same arm, so only route_reason+ # separates them. Reading a provider outage as flag_bucket_control mixes runs that were never+ # randomized into the control cell of every per-arm comparison, which is the one thing this field+ # exists to prevent. Only reachable through the strict resolver: the boolean one swallows the+ # error into None and bool()s it to a plain False.+ provider = _FakeExperimentationProvider(strict_error_flags={WORKFLOW_TASK_V3_AB_FLAG})+ resolution = await _resolve(scoped_context, provider, workflow_run_id="wr_ab_down", ineligibility_reason=None)++ assert scoped_context.workflow_block_engine_override is None+ assert _make_block(TaskBlock, label="ab_down").resolve_engine("wr_ab_down") == RunEngine.skyvern_v1+ assert resolution.log["route_reason"] == WorkflowBlockEngineRouteReason.flag_error+++async def test_an_ab_flag_local_evaluation_cannot_answer_is_control_but_not_a_bucket(+ scoped_context: SkyvernContext,+) -> None:+ # The third A/B answer: None from local evaluation is neither a failure nor a bucket. A key the+ # snapshot has no row for, or a condition local evaluation cannot answer, leaves the run on+ # control without it ever having been randomized -- and the runbook builds this rule's control+ # cell out of flag_bucket_control, so labelling it that way pours never-randomized runs into+ # every per-arm comparison, the same defect flag_error exists to prevent. The arm is unchanged.+ provider = _FakeExperimentationProvider(+ {WORKFLOW_TASK_V3_AB_FLAG: True}, unresolvable_flags={WORKFLOW_TASK_V3_AB_FLAG}+ )+ resolution = await _resolve(scoped_context, provider, workflow_run_id="wr_ab_undefined", ineligibility_reason=None)++ assert scoped_context.workflow_block_engine_override is None+ assert resolution.log["route_reason"] == WorkflowBlockEngineRouteReason.flag_undefineddef test_script_run_is_never_eligible() -> None:@@ -336,6 +508,8 @@ async def test_noop_provider_never_queried_and_leaves_engine_unchanged(scoped_coworkflow_run_id="wr_noop",organization_id="org_1",workflow_permanent_id="wpid_1",+ workflow_status=WorkflowStatus.published,+ trigger_type=WorkflowRunTriggerType.api,ineligibility_reason=None,)@@ -465,25 +639,35 @@ async def test_base_task_block_execute_dispatches_with_the_resolved_engine(scopeassert captured["engine"] == RunEngine.skyvern_v3-async def test_execute_workflow_blocks_pins_the_context_execute_safe_reads_from(- scoped_context: SkyvernContext,+async def _persisted_engine_from_execute_workflow_blocks(monkeypatch: pytest.MonkeyPatch,-) -> None:+ provider: BaseExperimentationProvider,+ *,+ workflow_status: WorkflowStatus = WorkflowStatus.published,+ billing_tier: BillingTier = BillingTier.UNKNOWN,+ first_version_created_at: datetime | None = None,+ trigger_type: WorkflowRunTriggerType | None = WorkflowRunTriggerType.api,+) -> RunEngine:"""Drives the real WorkflowService._execute_workflow_blocks -> _execute_single_block ->- Block.execute_safe chain for a single eligible TaskBlock, forcing only the experimentation- provider. If resolve_workflow_block_engine_arm pinned the arm on a different SkyvernContext- object than the one execute_safe's resolve_engine() reads back from, this fails: the- persisted engine would be skyvern_v1 instead of skyvern_v3.+ Block.execute_safe chain for a single eligible TaskBlock and returns the engine it persisted,+ forcing only the experimentation provider, the billing tier and the workflow's birth timestamp.+ Everything the arm decision depends on is therefore supplied by the real caller, not by the+ resolver's own test seam."""- provider = _FakeExperimentationProvider({WORKFLOW_TASK_V3_AB_FLAG: True})monkeypatch.setattr(app, "EXPERIMENTATION_PROVIDER", provider)+ monkeypatch.setattr(app.AGENT_FUNCTION, "resolve_billing_tier", AsyncMock(return_value=billing_tier))+ monkeypatch.setattr(+ app.DATABASE.workflows,+ "get_workflow_permanent_id_created_at",+ AsyncMock(return_value=first_version_created_at),+ )block = _make_block(TaskBlock, label="e2e_block")workflow = MagicMock()workflow.workflow_definition.blocks = [block]workflow.workflow_definition.version = 1workflow.workflow_definition.finally_block_label = None+ workflow.status = workflow_statusworkflow_run = MagicMock()workflow_run.workflow_run_id = "wr_e2e"@@ -491,6 +675,7 @@ async def test_execute_workflow_blocks_pins_the_context_execute_safe_reads_from(workflow_run.organization_id = "org_e2e"workflow_run.run_with = Noneworkflow_run.retried_from_workflow_run_id = None+ workflow_run.trigger_type = trigger_typeorganization = MagicMock()organization.organization_id = "org_e2e"@@ -527,7 +712,91 @@ async def test_execute_workflow_blocks_pins_the_context_execute_safe_reads_from(organization=organization,)- assert captured["engine"] == RunEngine.skyvern_v3+ return captured["engine"]+++async def test_execute_workflow_blocks_pins_the_context_execute_safe_reads_from(+ scoped_context: SkyvernContext,+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ """If resolve_workflow_block_engine_arm pinned the arm on a different SkyvernContext object+ than the one execute_safe's resolve_engine() reads back from, this fails: the persisted engine+ would be skyvern_v1 instead of skyvern_v3.+ """+ provider = _FakeExperimentationProvider({WORKFLOW_TASK_V3_AB_FLAG: True})++ engine = await _persisted_engine_from_execute_workflow_blocks(monkeypatch, provider)++ assert engine == RunEngine.skyvern_v3+++ ("workflow_status", "expected_engine"),+ [+ (WorkflowStatus.auto_generated, RunEngine.skyvern_v1),+ (WorkflowStatus.published, RunEngine.skyvern_v3),+ ],+)+async def test_execute_workflow_blocks_hands_the_resolver_the_running_versions_status(+ scoped_context: SkyvernContext,+ monkeypatch: pytest.MonkeyPatch,+ v3_default_cutoff: datetime,+ workflow_status: WorkflowStatus,+ expected_engine: RunEngine,+) -> None:+ # The per-call exclusion is only as real as the status the caller passes. Both cases are the same+ # new self-serve workflow with the percentage knob off, so the only thing deciding the engine is+ # workflow.status reaching the resolver: a caller that stopped passing it, or passed the+ # workflow_run instead, would take the login/download/credential-test/SDK endpoints' per-call+ # workflows onto v3 with it.+ provider = _FakeExperimentationProvider({WORKFLOW_TASK_V3_AB_FLAG: False})++ engine = await _persisted_engine_from_execute_workflow_blocks(+ monkeypatch,+ provider,+ workflow_status=workflow_status,+ billing_tier=BillingTier.SELF_SERVE,+ first_version_created_at=v3_default_cutoff.replace(tzinfo=None) + timedelta(days=1),+ )++ assert engine == expected_engine+++ ("trigger_type", "expected_engine"),+ [+ (WorkflowRunTriggerType.job_recipe_apply, RunEngine.skyvern_v1),+ (WorkflowRunTriggerType.job_recipe_extract, RunEngine.skyvern_v1),+ (WorkflowRunTriggerType.api, RunEngine.skyvern_v3),+ ],+)+async def test_execute_workflow_blocks_hands_the_resolver_the_runs_trigger_type(+ scoped_context: SkyvernContext,+ monkeypatch: pytest.MonkeyPatch,+ v3_default_cutoff: datetime,+ trigger_type: WorkflowRunTriggerType,+ expected_engine: RunEngine,+) -> None:+ # The job-recipe endpoints run a brand-new published workflow_permanent_id per request, so the+ # status the caller passes cannot exclude them and only the run's trigger kind can. All three+ # cases are the same new self-serve published workflow with the percentage knob off, so the only+ # thing deciding the engine is workflow_run.trigger_type reaching the resolver: a caller that+ # stopped passing it would force-route every recipe call to v3, unrandomized.+ provider = _FakeExperimentationProvider({WORKFLOW_TASK_V3_AB_FLAG: False})++ engine = await _persisted_engine_from_execute_workflow_blocks(+ monkeypatch,+ provider,+ workflow_status=WorkflowStatus.published,+ billing_tier=BillingTier.SELF_SERVE,+ first_version_created_at=v3_default_cutoff.replace(tzinfo=None) + timedelta(days=1),+ trigger_type=trigger_type,+ )++ assert engine == expected_enginedef _eligible_mix_blocks() -> list[BaseTaskBlock]:@@ -668,6 +937,8 @@ async def test_resolver_logs_the_ineligibility_reason(scoped_context: SkyvernConworkflow_run_id="wr_logged",organization_id="org_1",workflow_permanent_id="wpid_1",+ workflow_status=WorkflowStatus.published,+ trigger_type=WorkflowRunTriggerType.api,ineligibility_reason=V3AbIneligibleReason.pinned_engine,)@@ -834,6 +1105,8 @@ async def test_arm_offers_the_billing_tier_to_the_ab_flag_but_not_the_kill_switcworkflow_run_id="wr_tier",organization_id="org_1",workflow_permanent_id="wpid_1",+ workflow_status=WorkflowStatus.published,+ trigger_type=WorkflowRunTriggerType.api,ineligibility_reason=None,)@@ -864,6 +1137,8 @@ async def test_a_run_that_never_reaches_the_ab_pays_for_no_tier_lookup(scoped_coworkflow_run_id=run_id,organization_id="org_1",workflow_permanent_id="wpid_1",+ workflow_status=WorkflowStatus.published,+ trigger_type=WorkflowRunTriggerType.api,ineligibility_reason=reason,)tier.assert_not_awaited()@@ -885,9 +1160,474 @@ async def test_a_failing_billing_tier_lookup_cannot_decide_the_arm(scoped_contexworkflow_run_id="wr_tier_down",organization_id="org_1",workflow_permanent_id="wpid_1",+ workflow_status=WorkflowStatus.published,+ trigger_type=WorkflowRunTriggerType.api,ineligibility_reason=None,)assert scoped_context.workflow_block_engine_override == RunEngine.skyvern_v3by_flag = {flag: properties or {} for flag, _distinct_id, properties in provider.calls}assert by_flag[WORKFLOW_TASK_V3_AB_FLAG][BILLING_TIER_PROPERTY] == "unknown"+++async def test_new_self_serve_workflow_defaults_to_v3_without_consulting_the_ab_flag(+ scoped_context: SkyvernContext,+ v3_default_cutoff: datetime,+) -> None:+ # The whole point of the rule: the percentage knob is bypassed, so a flag sitting at 0% (or a+ # bucket that landed on control) cannot take a new self-serve workflow back to v1.+ provider = _FakeExperimentationProvider(+ {WORKFLOW_TASK_V3_AB_FLAG: False, TASK_V3_NEW_WORKFLOW_DEFAULT_ROLLOUT_FLAG: True}+ )+ run_id = "wr_new_self_serve_enrolled"+ resolution = await _resolve(+ scoped_context,+ provider,+ workflow_run_id=run_id,+ ineligibility_reason=None,+ billing_tier=BillingTier.SELF_SERVE,+ first_version_created_at=v3_default_cutoff.replace(tzinfo=None) + timedelta(seconds=1),+ )++ assert scoped_context.workflow_block_engine_override == RunEngine.skyvern_v3+ assert _make_block(TaskBlock, label="new_wf").resolve_engine(run_id) == RunEngine.skyvern_v3+ # The percentage knob is still never consulted for an enrolled run.+ assert _consulted_flags(provider) == [DISABLE_TASK_V3_FLAG, TASK_V3_NEW_WORKFLOW_DEFAULT_ROLLOUT_FLAG]+ assert resolution.log["route_reason"] == WorkflowBlockEngineRouteReason.new_self_serve_workflow_default+ assert resolution.log["new_workflow_default_rollout"] is True+ assert resolution.log["new_workflow_default_rollout_resolution"] == NewWorkflowDefaultRollout.enrolled+ assert resolution.log["billing_tier"] == BillingTier.SELF_SERVE.value+++ ("ab_flag", "expected_override", "expected_reason"),+ [+ (True, RunEngine.skyvern_v3, WorkflowBlockEngineRouteReason.flag_bucket_treatment),+ (False, None, WorkflowBlockEngineRouteReason.flag_bucket_control),+ ],+)+async def test_an_unenrolled_new_workflow_is_bucketed_by_the_ab_as_if_the_rule_did_not_exist(+ scoped_context: SkyvernContext,+ v3_default_cutoff: datetime,+ ab_flag: bool,+ expected_override: RunEngine | None,+ expected_reason: WorkflowBlockEngineRouteReason,+) -> None:+ # A conclusive False is what an operator's fastest off-gesture produces: posthog's local evaluator+ # answers False for an INACTIVE flag before it reads any filter, and the same value comes back for+ # a run outside the percentage or excluded by a condition. Enrolling on it would put 100% of this+ # population on v3 with the control cell zeroed at the exact moment someone switched the rollout+ # off, and the log would read like a rule that was not firing. So False leaves the run in the+ # ordinary bucketing, both ways -- which is also where this population's concurrent control and+ # its scoped kill come from, the cutoff being a setting that needs a restart and a condition on+ # WORKFLOW_TASK_V3_AB being unable to reach a run that never evaluates it.+ provider = _FakeExperimentationProvider(+ {WORKFLOW_TASK_V3_AB_FLAG: ab_flag, TASK_V3_NEW_WORKFLOW_DEFAULT_ROLLOUT_FLAG: False}+ )+ resolution = await _resolve(+ scoped_context,+ provider,+ workflow_run_id=f"wr_not_enrolled_{ab_flag}",+ ineligibility_reason=None,+ billing_tier=BillingTier.SELF_SERVE,+ first_version_created_at=v3_default_cutoff.replace(tzinfo=None) + timedelta(days=1),+ )++ assert scoped_context.workflow_block_engine_override == expected_override+ assert resolution.log["route_reason"] == expected_reason+ assert resolution.log["new_workflow_default_rollout"] is False+ assert resolution.log["new_workflow_default_rollout_resolution"] == NewWorkflowDefaultRollout.not_enrolled+ rollout_call = next(call for call in provider.calls if call[0] == TASK_V3_NEW_WORKFLOW_DEFAULT_ROLLOUT_FLAG)+ # Bucketed per run, and carrying the two properties a scoped condition is written against: holding+ # one organization or one workflow out of the rollout is what the runbook promises as the fast+ # lever.+ assert rollout_call[1] == f"wr_not_enrolled_{ab_flag}"+ assert rollout_call[2] == {+ "organization_id": "org_1",+ "workflow_permanent_id": "wpid_1",+ BILLING_TIER_PROPERTY: BillingTier.SELF_SERVE.value,+ }+++async def test_a_failing_rollout_evaluation_leaves_the_run_on_the_ab_path(+ scoped_context: SkyvernContext, v3_default_cutoff: datetime+) -> None:+ # An evaluation that raised is not an answer, and treating it as one would enrol a population+ # whose fastest kill is the flag that just failed. It also must not be reported as a real+ # "outside the percentage": the error resolution is what separates the two on the log. Only+ # reachable through the strict resolver -- the boolean one swallows the error into None and+ # bool()s it to False.+ provider = _FakeExperimentationProvider(+ {WORKFLOW_TASK_V3_AB_FLAG: False}, strict_error_flags={TASK_V3_NEW_WORKFLOW_DEFAULT_ROLLOUT_FLAG}+ )+ resolution = await _resolve(+ scoped_context,+ provider,+ workflow_run_id="wr_rollout_down",+ ineligibility_reason=None,+ billing_tier=BillingTier.SELF_SERVE,+ first_version_created_at=v3_default_cutoff.replace(tzinfo=None) + timedelta(days=1),+ )++ assert scoped_context.workflow_block_engine_override is None+ assert resolution.log["route_reason"] == WorkflowBlockEngineRouteReason.flag_bucket_control+ assert resolution.log["new_workflow_default_rollout"] is False+ assert resolution.log["new_workflow_default_rollout_resolution"] == NewWorkflowDefaultRollout.error+ assert WORKFLOW_TASK_V3_AB_FLAG in _consulted_flags(provider)+ # A rollout that stopped answering must not be silent: an operator reading a quiet+ # new_self_serve_workflow_default needs this warning and the resolution field above to tell a+ # broken evaluation from a rollout nobody enabled.+ assert [call.kwargs.get("workflow_run_id") for call in resolution.warnings] == ["wr_rollout_down"]+++ ("ab_flag", "expected_override", "expected_reason"),+ [+ (True, RunEngine.skyvern_v3, WorkflowBlockEngineRouteReason.flag_bucket_treatment),+ (False, None, WorkflowBlockEngineRouteReason.flag_bucket_control),+ ],+)+async def test_an_unresolvable_rollout_flag_leaves_the_run_on_the_ab_path(+ scoped_context: SkyvernContext,+ v3_default_cutoff: datetime,+ ab_flag: bool,+ expected_override: RunEngine | None,+ expected_reason: WorkflowBlockEngineRouteReason,+) -> None:+ # The arm-resolving processes evaluate PostHog locally against a snapshot, which answers None --+ # not False, not an error -- for a flag key it has no row for; so does a cutoff set before anyone+ # created the flag, and so does a condition local evaluation cannot answer. Enrolling on that+ # would make the cutoff alone a 100%-v3 cohort with no concurrent control, and would bypass the+ # organization exclusions this rollout's own conditions carry. So an unresolved flag leaves the+ # run in the ordinary bucketing, both ways -- and the resolution field still separates "no flag"+ # from a rollout that answered, which is what an operator reads after an enable.+ provider = _FakeExperimentationProvider(+ {WORKFLOW_TASK_V3_AB_FLAG: ab_flag},+ unresolvable_flags={TASK_V3_NEW_WORKFLOW_DEFAULT_ROLLOUT_FLAG},+ )+ resolution = await _resolve(+ scoped_context,+ provider,+ workflow_run_id=f"wr_rollout_undefined_{ab_flag}",+ ineligibility_reason=None,+ billing_tier=BillingTier.SELF_SERVE,+ first_version_created_at=v3_default_cutoff.replace(tzinfo=None) + timedelta(days=1),+ )++ assert scoped_context.workflow_block_engine_override == expected_override+ assert resolution.log["route_reason"] == expected_reason+ assert resolution.log["new_workflow_default_rollout"] is False+ assert resolution.log["new_workflow_default_rollout_resolution"] == NewWorkflowDefaultRollout.undefined+ assert WORKFLOW_TASK_V3_AB_FLAG in _consulted_flags(provider)+++async def test_a_new_version_of_an_older_workflow_is_not_a_new_workflow(+ scoped_context: SkyvernContext, v3_default_cutoff: datetime+) -> None:+ # A workflow saved again after the cutoff keeps an old birthday. _resolve wires the latest+ # version at _LATEST_VERSION_CREATED_AT, on the far side of the cutoff, so a resolver that read+ # the version this run executes would enrol this workflow and flip the arm below to treatment --+ # which is the regression that would otherwise take every long-lived workflow onto v3 at once.+ provider = _FakeExperimentationProvider({WORKFLOW_TASK_V3_AB_FLAG: False})+ resolution = await _resolve(+ scoped_context,+ provider,+ workflow_run_id="wr_old_wf",+ ineligibility_reason=None,+ billing_tier=BillingTier.SELF_SERVE,+ first_version_created_at=v3_default_cutoff.replace(tzinfo=None) - timedelta(days=30),+ )++ assert scoped_context.workflow_block_engine_override is None+ assert DISABLE_TASK_V3_FLAG in [flag for flag, _distinct_id, _properties in provider.calls]+ assert WORKFLOW_TASK_V3_AB_FLAG in [flag for flag, _distinct_id, _properties in provider.calls]+ assert resolution.log["route_reason"] == WorkflowBlockEngineRouteReason.flag_bucket_control+ assert TASK_V3_NEW_WORKFLOW_DEFAULT_ROLLOUT_FLAG not in _consulted_flags(provider)+ # Asserted here rather than inside the fake, whose AssertionError the rule's catch-all would+ # swallow into "not a new workflow": the birth timestamp is read once, scoped to this run's own+ # organization. The read itself is the min over every version of the id including deleted ones,+ # so a deleted or copilot-test first version cannot make a long-lived workflow look unborn.+ resolution.birth_reads.assert_awaited_once_with("wpid_1", "org_1")+++async def test_a_per_call_auto_generated_workflow_is_never_a_new_workflow(+ scoped_context: SkyvernContext, v3_default_cutoff: datetime+) -> None:+ # /v1/run/tasks/login, /v1/run/tasks/download_files, the two credential test-login endpoints and+ # the SDK run endpoint each mint a fresh workflow_permanent_id per call and run it while it is+ # still auto_generated, holding a v3-eligible block. Every one of those is born after any cutoff,+ # so without this exclusion setting the cutoff moves all of that standing API traffic onto v3 at+ # once instead of only the workflows customers keep.+ provider = _FakeExperimentationProvider({WORKFLOW_TASK_V3_AB_FLAG: False})+ resolution = await _resolve(+ scoped_context,+ provider,+ workflow_run_id="wr_auto_generated",+ ineligibility_reason=None,+ billing_tier=BillingTier.SELF_SERVE,+ first_version_created_at=v3_default_cutoff.replace(tzinfo=None) + timedelta(days=1),+ workflow_status=WorkflowStatus.auto_generated,+ )++ assert scoped_context.workflow_block_engine_override is None+ assert WORKFLOW_TASK_V3_AB_FLAG in _consulted_flags(provider)+ assert TASK_V3_NEW_WORKFLOW_DEFAULT_ROLLOUT_FLAG not in _consulted_flags(provider)+ assert resolution.log["route_reason"] == WorkflowBlockEngineRouteReason.flag_bucket_control+ assert resolution.log["new_workflow_default_rollout_resolution"] is None+ # A status the rule cannot use must not pay for the workflow read either.+ resolution.birth_reads.assert_not_awaited()+++ "trigger_type",+ [WorkflowRunTriggerType.job_recipe_apply, WorkflowRunTriggerType.job_recipe_extract],+)+async def test_a_per_call_recipe_run_is_never_a_new_workflow(+ scoped_context: SkyvernContext,+ v3_default_cutoff: datetime,+ trigger_type: WorkflowRunTriggerType,+) -> None:+ # The job-recipe endpoints also mint a fresh workflow_permanent_id per request and run it in that+ # same request, but the definition they build carries no status and so is born published: the+ # auto_generated exclusion cannot see it, and every one of those ids is born after any cutoff.+ # One run per permanent id also means an enrolled recipe call has no control run of the same+ # workflow to be matched against, so the trigger kind is what keeps this standing API traffic+ # randomized by the A/B instead of force-routed.+ provider = _FakeExperimentationProvider({WORKFLOW_TASK_V3_AB_FLAG: False})+ resolution = await _resolve(+ scoped_context,+ provider,+ workflow_run_id=f"wr_{trigger_type.value}",+ ineligibility_reason=None,+ billing_tier=BillingTier.SELF_SERVE,+ first_version_created_at=v3_default_cutoff.replace(tzinfo=None) + timedelta(days=1),+ workflow_status=WorkflowStatus.published,+ trigger_type=trigger_type,+ )++ assert scoped_context.workflow_block_engine_override is None+ assert WORKFLOW_TASK_V3_AB_FLAG in _consulted_flags(provider)+ assert TASK_V3_NEW_WORKFLOW_DEFAULT_ROLLOUT_FLAG not in _consulted_flags(provider)+ assert resolution.log["route_reason"] == WorkflowBlockEngineRouteReason.flag_bucket_control+ assert resolution.log["new_workflow_default_rollout_resolution"] is None+ # A trigger the rule cannot use must not pay for the workflow read either.+ resolution.birth_reads.assert_not_awaited()+++async def test_a_workflow_born_auto_generated_and_kept_by_the_customer_is_a_new_workflow(+ scoped_context: SkyvernContext, v3_default_cutoff: datetime+) -> None:+ # POST /workflows/create-from-prompt -- the prompt box, the product's main new-workflow path --+ # writes version 1 as auto_generated whenever publish_workflow is false, which is its default, and+ # then drops the customer into the editor on that same workflow_permanent_id; an editor save+ # writes the next version as published. Excluding on the BIRTH version's status would therefore+ # exclude every prompt-created workflow forever, including the ones a customer edits and re-runs+ # for months, and the rule's whole stated purpose would be defeated for the path that creates most+ # of its population. The exclusion is about the version this run executes, not about how the id+ # was born.+ provider = _FakeExperimentationProvider({WORKFLOW_TASK_V3_AB_FLAG: False})+ resolution = await _resolve(+ scoped_context,+ provider,+ workflow_run_id="wr_prompt_created_kept",+ ineligibility_reason=None,+ billing_tier=BillingTier.SELF_SERVE,+ first_version_created_at=v3_default_cutoff.replace(tzinfo=None) + timedelta(days=1),+ first_version_status=WorkflowStatus.auto_generated,+ workflow_status=WorkflowStatus.published,+ )++ assert scoped_context.workflow_block_engine_override == RunEngine.skyvern_v3+ assert resolution.log["route_reason"] == WorkflowBlockEngineRouteReason.new_self_serve_workflow_default+ assert WORKFLOW_TASK_V3_AB_FLAG not in _consulted_flags(provider)+++ ("first_version_created_at", "expect_v3_default"),+ [+ (datetime(2026, 9, 14, 23, 59, 59), False),+ # Naive rows come from datetime.utcnow(): the cutoff is aware, so a resolver that compared+ # them directly would raise and lose every enrolment. At-or-after includes the instant itself.+ (datetime(2026, 9, 15, 0, 0, 0), True),+ (datetime(2026, 9, 15, 0, 0, 1), True),+ (datetime(2026, 9, 15, 0, 0, 1, tzinfo=UTC), True),+ # 2026-09-14 21:00 at UTC-4 is 2026-09-15 01:00 UTC, i.e. after the cutoff despite the date.+ (datetime(2026, 9, 14, 21, 0, 0, tzinfo=timezone(timedelta(hours=-4))), True),+ ],+)+async def test_cutoff_compares_aware_and_naive_first_version_timestamps_in_utc(+ scoped_context: SkyvernContext,+ v3_default_cutoff: datetime,+ first_version_created_at: datetime,+ expect_v3_default: bool,+) -> None:+ provider = _FakeExperimentationProvider({WORKFLOW_TASK_V3_AB_FLAG: False})+ resolution = await _resolve(+ scoped_context,+ provider,+ workflow_run_id=f"wr_cutoff_{first_version_created_at.isoformat()}",+ ineligibility_reason=None,+ billing_tier=BillingTier.SELF_SERVE,+ first_version_created_at=first_version_created_at,+ )++ expected_override = RunEngine.skyvern_v3 if expect_v3_default else None+ assert scoped_context.workflow_block_engine_override == expected_override+ expected_reason = (+ WorkflowBlockEngineRouteReason.new_self_serve_workflow_default+ if expect_v3_default+ else WorkflowBlockEngineRouteReason.flag_bucket_control+ )+ assert resolution.log["route_reason"] == expected_reason++[email protected](not hasattr(time, "tzset"), reason="TZ cannot be changed in-process on this platform")+def test_a_naive_created_at_is_read_as_utc_not_as_local_time(monkeypatch: pytest.MonkeyPatch) -> None:+ # "Naive means UTC" and the classic astimezone() spelling that means "naive means local" are the+ # same function on a UTC machine, which CI and every container is -- so neither the cases above+ # nor any assertion made under the ambient zone can tell them apart. workflows.created_at is+ # datetime.utcnow(), so reading it as local time misclassifies every workflow born within the+ # process's TZ offset of the cutoff, silently enrolling or excluding runs that have no control arm.+ monkeypatch.setenv("TZ", "America/New_York")+ time.tzset()+ try:+ assert _as_utc(datetime(2026, 9, 15, 12)) == datetime(2026, 9, 15, 12, tzinfo=UTC)+ finally:+ monkeypatch.undo()+ time.tzset()++[email protected]("billing_tier", [BillingTier.ENTERPRISE, BillingTier.UNKNOWN])+async def test_only_the_self_serve_tier_gets_the_new_workflow_default(+ scoped_context: SkyvernContext, v3_default_cutoff: datetime, billing_tier: BillingTier+) -> None:+ # UNKNOWN is a failed lookup, so it must fall through with enterprise rather than be force-routed.+ provider = _FakeExperimentationProvider({WORKFLOW_TASK_V3_AB_FLAG: True})+ resolution = await _resolve(+ scoped_context,+ provider,+ workflow_run_id=f"wr_tier_{billing_tier.value}",+ ineligibility_reason=None,+ billing_tier=billing_tier,+ first_version_created_at=v3_default_cutoff.replace(tzinfo=None) + timedelta(days=1),+ )++ assert WORKFLOW_TASK_V3_AB_FLAG in [flag for flag, _distinct_id, _properties in provider.calls]+ assert resolution.log["route_reason"] == WorkflowBlockEngineRouteReason.flag_bucket_treatment+ # A tier the rule cannot use must not pay for the workflow read either.+ resolution.birth_reads.assert_not_awaited()+ assert TASK_V3_NEW_WORKFLOW_DEFAULT_ROLLOUT_FLAG not in _consulted_flags(provider)+++async def test_kill_switch_wins_over_the_new_workflow_default(+ scoped_context: SkyvernContext, v3_default_cutoff: datetime+) -> None:+ provider = _FakeExperimentationProvider({WORKFLOW_TASK_V3_AB_FLAG: True, DISABLE_TASK_V3_FLAG: True})+ resolution = await _resolve(+ scoped_context,+ provider,+ workflow_run_id="wr_new_but_killed",+ ineligibility_reason=None,+ billing_tier=BillingTier.SELF_SERVE,+ first_version_created_at=v3_default_cutoff.replace(tzinfo=None) + timedelta(days=1),+ )++ assert scoped_context.workflow_block_engine_override is None+ assert resolution.log["route_reason"] == WorkflowBlockEngineRouteReason.disabled+ resolution.birth_reads.assert_not_awaited()+ assert TASK_V3_NEW_WORKFLOW_DEFAULT_ROLLOUT_FLAG not in _consulted_flags(provider)+++async def test_no_cutoff_leaves_every_workflow_on_the_flag_path(scoped_context: SkyvernContext) -> None:+ # Deliberately does not request v3_default_cutoff: the autouse pin holds the setting at None.+ provider = _FakeExperimentationProvider({WORKFLOW_TASK_V3_AB_FLAG: False})+ resolution = await _resolve(+ scoped_context,+ provider,+ workflow_run_id="wr_no_cutoff",+ ineligibility_reason=None,+ billing_tier=BillingTier.SELF_SERVE,+ first_version_created_at=datetime(2030, 1, 1),+ )++ assert scoped_context.workflow_block_engine_override is None+ assert resolution.log["route_reason"] == WorkflowBlockEngineRouteReason.flag_bucket_control+ resolution.birth_reads.assert_not_awaited()+ assert TASK_V3_NEW_WORKFLOW_DEFAULT_ROLLOUT_FLAG not in _consulted_flags(provider)+++async def test_ineligible_run_is_never_defaulted_to_v3(+ scoped_context: SkyvernContext, v3_default_cutoff: datetime+) -> None:+ # Run-level eligibility is about whether the run can be rerouted at all, so the rule changes the+ # arm decision only: a script run or a pinned block still executes as authored.+ provider = _FakeExperimentationProvider({WORKFLOW_TASK_V3_AB_FLAG: True})+ resolution = await _resolve(+ scoped_context,+ provider,+ workflow_run_id="wr_new_but_ineligible",+ ineligibility_reason=V3AbIneligibleReason.script_run,+ billing_tier=BillingTier.SELF_SERVE,+ first_version_created_at=v3_default_cutoff.replace(tzinfo=None) + timedelta(days=1),+ )++ assert scoped_context.workflow_block_engine_override is None+ assert provider.calls == []+ assert resolution.log["route_reason"] == WorkflowBlockEngineRouteReason.ineligible+ resolution.birth_reads.assert_not_awaited()+ assert TASK_V3_NEW_WORKFLOW_DEFAULT_ROLLOUT_FLAG not in _consulted_flags(provider)+++async def test_a_failing_first_version_read_falls_back_to_the_flag_path(+ scoped_context: SkyvernContext, v3_default_cutoff: datetime+) -> None:+ # The new rule must not be able to take the run off the arm it would have had: a database blip+ # falls back to the A/B, not to control, and never forces v3.+ provider = _FakeExperimentationProvider({WORKFLOW_TASK_V3_AB_FLAG: True})+ resolution = await _resolve(+ scoped_context,+ provider,+ workflow_run_id="wr_first_version_down",+ ineligibility_reason=None,+ billing_tier=BillingTier.SELF_SERVE,+ first_version_error=RuntimeError("workflows read failed"),+ )++ assert scoped_context.workflow_block_engine_override == RunEngine.skyvern_v3+ assert resolution.log["route_reason"] == WorkflowBlockEngineRouteReason.flag_bucket_treatment+ # The fallback is silent in the arm it produces, so the swallowed read has to be visible+ # somewhere: a rule that stops enrolling because a query broke must not look like a rule nobody+ # set the cutoff for.+ assert [call.kwargs.get("workflow_permanent_id") for call in resolution.warnings] == ["wpid_1"]+++async def test_a_missing_first_version_falls_back_to_the_flag_path(+ scoped_context: SkyvernContext, v3_default_cutoff: datetime+) -> None:+ provider = _FakeExperimentationProvider({WORKFLOW_TASK_V3_AB_FLAG: False})+ resolution = await _resolve(+ scoped_context,+ provider,+ workflow_run_id="wr_first_version_missing",+ ineligibility_reason=None,+ billing_tier=BillingTier.SELF_SERVE,+ first_version_created_at=None,+ )++ assert scoped_context.workflow_block_engine_override is None+ assert resolution.log["route_reason"] == WorkflowBlockEngineRouteReason.flag_bucket_control