Skip to content

Automate browser based workflows with AI

Run a new non-enterprise workflow's task blocks on Task V3 by default (SKY-16293) (#8545)

Shuchang Zhengfcc14e82776d
diff --git a/skyvern/config.py b/skyvern/config.py
index 9df689538..80266bf4a 100644
--- a/skyvern/config.py
+++ b/skyvern/config.py
@@ -1,6 +1,7 @@
import logging
import os
import platform
+from datetime import datetime
from enum import StrEnum
from pathlib import Path
from typing import Any, Literal
@@ -557,6 +558,20 @@ class Settings(BaseSettings):
# executes only in the sandboxed runner and is withheld whenever that runner is unavailable --
# there is no in-process execution path to fall back to.
TASK_V3_CODE_TOOL_SURFACE: Literal["off", "add", "replace"] = "off"
+ # Workflows whose permanent id was born at or after this instant run their task blocks on Task V3
+ # when the organization resolves to the self-serve billing tier (an unknown tier is not enrolled),
+ # bypassing WORKFLOW_TASK_V3_AB. None disables the rule (the OSS default). Setting it is not
+ # enough on its own: the rule fires only for runs whose TASK_V3_NEW_WORKFLOW_DEFAULT_ROLLOUT
+ # evaluation returns a conclusive true, so the cutoff stays inert until that flag enrols someone.
+ # Every off-state of that flag -- disabled, deleted, 0%, an excluding condition, an evaluation
+ # that raised, no flag provider at all -- leaves the run on the A/B, so there is no off-state that
+ # enrols. A naive value is read as UTC. Per-call platform workflows are excluded twice over -- by
+ # an auto_generated executing version, which covers the login, download_files, credential
+ # test-login and SDK endpoints, and by a per-call trigger kind, which covers the job recipe
+ # endpoints because those build a published definition. Enrolled runs are not randomized, so every
+ # per-arm read must exclude them by route_reason and read them against the unenrolled control cell
+ # instead.
+ TASK_V3_DEFAULT_ENGINE_WORKFLOW_CUTOFF: datetime | None = None
# VOLCENGINE (Doubao)
ENABLE_VOLCENGINE: bool = False
@@ -1012,6 +1027,16 @@ class Settings(BaseSettings):
return None
return value
+ @field_validator("TASK_V3_DEFAULT_ENGINE_WORKFLOW_CUTOFF", mode="before")
+ @classmethod
+ def _task_v3_default_engine_workflow_cutoff_off_sentinels(cls, value: Any) -> Any:
+ # This setting is the rule's settings-side kill path, and blanking an already-set env var is
+ # how it gets turned off mid-incident. Without this, every off-spelling fails datetime
+ # validation and crashes the process at import instead of disabling the rule.
+ if value is None or str(value).strip().lower() in ("", "0", "none", "null", "off", "disabled"):
+ return None
+ return value
+
@field_validator("WORKER_STALL_DUMP_SECONDS", mode="before")
@classmethod
def _worker_stall_dump_seconds_or_default(cls, value: Any) -> int:
diff --git a/skyvern/forge/sdk/experimentation/workflow_block_engine.py b/skyvern/forge/sdk/experimentation/workflow_block_engine.py
index ed8d202b9..bd96c0ce8 100644
--- a/skyvern/forge/sdk/experimentation/workflow_block_engine.py
+++ b/skyvern/forge/sdk/experimentation/workflow_block_engine.py
@@ -1,14 +1,19 @@
from __future__ import annotations
+from datetime import UTC, datetime
+from enum import StrEnum
from typing import TYPE_CHECKING
import structlog
+from skyvern.config import settings
from skyvern.forge import app
from skyvern.forge.sdk.core import skyvern_context
+from skyvern.forge.sdk.db.enums import JOB_RECIPE_WORKFLOW_RUN_TRIGGER_TYPES, WorkflowRunTriggerType
from skyvern.forge.sdk.experimentation.billing_tier import BILLING_TIER_PROPERTY, BillingTier
-from skyvern.forge.sdk.experimentation.providers import NoOpExperimentationProvider
+from skyvern.forge.sdk.experimentation.providers import BaseExperimentationProvider, NoOpExperimentationProvider
from skyvern.schemas.run_enums import RunEngine
+from skyvern.schemas.workflows import WorkflowStatus
if TYPE_CHECKING:
# Runtime import would cycle: block.py imports workflow_block_engine_override from this module.
@@ -18,6 +23,63 @@ LOG = structlog.get_logger()
WORKFLOW_TASK_V3_AB_FLAG = "WORKFLOW_TASK_V3_AB"
DISABLE_TASK_V3_FLAG = "DISABLE_TASK_V3"
+TASK_V3_NEW_WORKFLOW_DEFAULT_ROLLOUT_FLAG = "TASK_V3_NEW_WORKFLOW_DEFAULT_ROLLOUT"
+
+# Trigger kinds whose run executes a workflow the platform minted for that one request rather than
+# one a customer built. The job-recipe endpoints are the whole set today: each call creates a fresh
+# workflow_permanent_id, runs it immediately, and never runs it again, so a per-workflow rule cannot
+# match one against a control run of the same workflow.
+PER_CALL_WORKFLOW_RUN_TRIGGER_TYPES: frozenset[WorkflowRunTriggerType] = JOB_RECIPE_WORKFLOW_RUN_TRIGGER_TYPES
+
+
+class WorkflowBlockEngineRouteReason(StrEnum):
+ """Why a workflow run landed on its arm, mirroring the bare-task RunRouteReason.
+
+ Both arms persist the same engine on a control run, so without this the log cannot separate a
+ bucketed control run from one that was never randomized -- and
+ ``new_self_serve_workflow_default`` runs were never randomized, so every per-arm read has to
+ exclude them by this field rather than by ``arm``. ``flag_bucket_control`` is therefore the
+ randomized control cell and nothing else: a run the A/B flag never answered for is
+ ``flag_undefined`` and a run whose evaluation raised is ``flag_error``, both on control.
+ """
+
+ flag_bucket_treatment = "flag_bucket_treatment"
+ flag_bucket_control = "flag_bucket_control"
+ new_self_serve_workflow_default = "new_self_serve_workflow_default"
+ ineligible = "ineligible"
+ disabled = "disabled"
+ flag_undefined = "flag_undefined"
+ flag_error = "flag_error"
+
+
+class NewWorkflowDefaultRollout(StrEnum):
+ """How ``TASK_V3_NEW_WORKFLOW_DEFAULT_ROLLOUT`` answered for a run the v3 default would enrol.
+
+ Only ``enrolled`` -- a conclusive True -- lets the rule fire. ``not_enrolled`` is a conclusive
+ False: the flag is inactive, the run fell outside the percentage, or a condition excluded it.
+ ``undefined`` means the key never resolved at all: nobody created it, or local evaluation has no
+ snapshot row for it yet. ``error`` means the evaluation raised. The last three are logged apart
+ from each other because only ``not_enrolled`` is a randomized cell -- a read that wants this
+ rule's control must pin that value, not the boolean.
+ """
+
+ enrolled = "enrolled"
+ not_enrolled = "not_enrolled"
+ undefined = "undefined"
+ error = "error"
+
+
+# Every resolution but a conclusive True leaves the run on the A/B path, so every off-state of the
+# flag -- inactive, deleted, 0%, a condition that excluded the run, an evaluation that raised -- turns
+# the rule off. An inactive flag is the one that has to be safe: posthog's local evaluator answers a
+# conclusive False for it (``_compute_flag_locally``, "if not active: return False"), which is the
+# gesture an operator reaches for mid-incident. Derived from the enum so a resolution added later
+# leaves the run on the A/B instead of enrolling it.
+_NOT_ENROLLED_RESOLUTIONS = frozenset(NewWorkflowDefaultRollout) - {NewWorkflowDefaultRollout.enrolled}
+
+
+def _rollout_enrols(rollout: NewWorkflowDefaultRollout | None) -> bool:
+ return rollout is not None and rollout not in _NOT_ENROLLED_RESOLUTIONS
async def task_v3_disabled(distinct_id: str, organization_id: str | None) -> bool:
@@ -45,12 +107,130 @@ async def _billing_tier_for_arm(organization_id: str | None) -> BillingTier:
return BillingTier.UNKNOWN
+def _as_utc(value: datetime) -> datetime:
+ # workflows.created_at is written by datetime.utcnow(), i.e. naive UTC, while the cutoff setting
+ # arrives aware or naive depending on how it was spelled in the environment.
+ return value.replace(tzinfo=UTC) if value.tzinfo is None else value.astimezone(UTC)
+
+
+async def _workflow_is_new_for_v3_default(workflow_permanent_id: str | None, organization_id: str | None) -> bool:
+ """Whether this permanent id was born at or after the v3-default cutoff.
+
+ Never raises and never answers True on a read it could not complete: the rule can only add
+ treatment, so any doubt has to leave the run on the arm the A/B would have given it.
+ """
+ cutoff = settings.TASK_V3_DEFAULT_ENGINE_WORKFLOW_CUTOFF
+ if cutoff is None or not workflow_permanent_id or not organization_id:
+ return False
+ try:
+ # The earliest version's timestamp, deleted versions included, is when the permanent id was
+ # born. Reading the version this run executes would instead enrol every long-lived workflow
+ # saved once after the cutoff.
+ born_at = await app.DATABASE.workflows.get_workflow_permanent_id_created_at(
+ workflow_permanent_id, organization_id
+ )
+ if born_at is None:
+ return False
+ # Inside the try with the read: a rule that raised here would reach the resolver's catch-all
+ # and send the run to control, which is a different arm than the A/B would have given it.
+ return _as_utc(born_at) >= _as_utc(cutoff)
+ except Exception:
+ LOG.warning(
+ "Failed to read the workflow's birth timestamp for the Task V3 default rule",
+ workflow_permanent_id=workflow_permanent_id,
+ exc_info=True,
+ )
+ return False
+
+
+async def _ab_flag_puts_run_in_treatment(
+ provider: BaseExperimentationProvider,
+ *,
+ workflow_run_id: str,
+ organization_id: str | None,
+ workflow_permanent_id: str | None,
+ billing_tier: BillingTier,
+) -> bool | None:
+ """Whether the percentage knob buckets this run into treatment, or ``None`` if it never answered.
+
+ ``resolve_feature_flag_strict`` rather than the boolean resolver so neither answer that is not a
+ bucket collapses into one: both arms leave the same engine on a control run, so a swallowed error
+ or an unresolved key would otherwise enter every per-arm read as a bucketed control run that was
+ never randomized. A raised evaluation propagates to the resolver's catch-all and is labelled
+ ``flag_error``; ``None`` -- the key is absent from the local snapshot, or carries a condition
+ local evaluation cannot answer -- is returned as itself and labelled ``flag_undefined``. The arm
+ is control for all three. Cached on the same (flag, distinct_id, properties) key the boolean
+ resolver was, in the provider's own strict map, so this stays one evaluation per run.
+ """
+ return await provider.resolve_feature_flag_strict(
+ WORKFLOW_TASK_V3_AB_FLAG,
+ workflow_run_id,
+ properties={
+ "organization_id": organization_id,
+ "workflow_permanent_id": workflow_permanent_id or "not_workflow",
+ BILLING_TIER_PROPERTY: billing_tier.value,
+ },
+ )
+
+
+async def _resolve_new_workflow_default_rollout(
+ provider: BaseExperimentationProvider,
+ *,
+ workflow_run_id: str,
+ organization_id: str | None,
+ workflow_permanent_id: str | None,
+ billing_tier: BillingTier,
+) -> NewWorkflowDefaultRollout:
+ """How the rollout flag answered for a run the new-workflow v3 default would otherwise enrol.
+
+ This flag is what enrols: the share it does not enrol continues down ``WORKFLOW_TASK_V3_AB``
+ exactly as if the rule did not exist, which is where this population's concurrent control comes
+ from. It is also the only lever that stops the rule for less than everybody -- the cutoff is a
+ setting and needs a restart, and a condition on ``WORKFLOW_TASK_V3_AB`` cannot reach a run that
+ never evaluates it. Turning it off de-enrols rather than forcing v1: an unenrolled run can still
+ be treated by the A/B, so ``DISABLE_TASK_V3`` is the lever for getting an organization off v3.
+
+ ``resolve_feature_flag_strict`` rather than the boolean resolver, so an evaluation that raised is
+ labelled rather than collapsed: both PostHog providers swallow one into ``None`` and ``bool()`` it
+ to ``False``, which would report an error as a real "outside the percentage".
+
+ Only a conclusive ``True`` enrols. ``False``, ``None`` and a raised evaluation all leave the run on
+ the A/B, so every off-gesture on this flag -- disable, delete, 0%, an excluding condition -- turns
+ the rule off, and the cutoff does nothing until the flag resolves ``True`` for someone. A
+ condition on it must be written on person properties: the arm-resolving processes evaluate
+ PostHog locally, and a cohort, group or flag-dependency condition resolves to ``None``, which
+ enrols nobody.
+ """
+ try:
+ resolved = await provider.resolve_feature_flag_strict(
+ TASK_V3_NEW_WORKFLOW_DEFAULT_ROLLOUT_FLAG,
+ workflow_run_id,
+ properties={
+ "organization_id": organization_id,
+ "workflow_permanent_id": workflow_permanent_id,
+ BILLING_TIER_PROPERTY: billing_tier.value,
+ },
+ )
+ except Exception:
+ LOG.warning(
+ "Failed to evaluate the new-workflow v3 default rollout; leaving this run on the A/B",
+ workflow_run_id=workflow_run_id,
+ exc_info=True,
+ )
+ return NewWorkflowDefaultRollout.error
+ if resolved is None:
+ return NewWorkflowDefaultRollout.undefined
+ return NewWorkflowDefaultRollout.enrolled if resolved else NewWorkflowDefaultRollout.not_enrolled
+
+
async def resolve_workflow_block_engine_arm(
context: skyvern_context.SkyvernContext,
*,
workflow_run_id: str,
organization_id: str | None,
workflow_permanent_id: str | None,
+ workflow_status: WorkflowStatus,
+ trigger_type: WorkflowRunTriggerType | None,
ineligibility_reason: V3AbIneligibleReason | None,
) -> None:
"""Resolve the workflow-block engine A/B once at execution start and pin the arm on the context.
@@ -61,6 +241,21 @@ async def resolve_workflow_block_engine_arm(
own workflow_run_id and its own definition -- re-resolves instead of inheriting an arm that was
never checked against its blocks. Control (no override) is the safe default and the outcome of
every failure.
+
+ A self-serve organization's workflow born at or after
+ ``settings.TASK_V3_DEFAULT_ENGINE_WORKFLOW_CUTOFF`` takes v3 without the percentage being
+ consulted, so those runs are marked ``new_self_serve_workflow_default`` for every per-arm read to
+ exclude. ``TASK_V3_NEW_WORKFLOW_DEFAULT_ROLLOUT``, read only for the runs that rule would enrol,
+ is what enrols them; the share it leaves alone continues down the A/B, which is where this
+ population's concurrent control and its scoped kill come from. The rule fires only on a
+ conclusive True, so the cutoff is inert until the flag resolves True for someone, and disabling,
+ deleting or zeroing the flag turns the rule off.
+
+ ``workflow_status`` is the status of the version this run executes and ``trigger_type`` is how
+ the run was launched. Together they are what separates a workflow a customer keeps from the
+ per-call throwaways several endpoints mint and run in one request: most of those run while still
+ auto_generated, and the job-recipe endpoints build a published definition, so neither test alone
+ covers them.
"""
if context.workflow_block_engine_resolved_run_id == workflow_run_id:
return
@@ -76,30 +271,74 @@ async def resolve_workflow_block_engine_arm(
override: RunEngine | None = None
run_is_eligible = ineligibility_reason is None
billing_tier: BillingTier | None = None
+ route_reason = WorkflowBlockEngineRouteReason.ineligible
+ rollout: NewWorkflowDefaultRollout | None = None
try:
if run_is_eligible:
# The kill switch, shared with the dispatch gate via task_v3_disabled so both
- # evaluations use the same cache key, wins over the experiment. A kill flipped
- # mid-run still takes effect at dispatch while these rows already read v3:
- # stopping the run wins over its attribution. Read FIRST, and before the billing
- # lookup below: a Redis or pooler incident is exactly when someone flips this, and
- # this lock is held throughout, so the kill switch must not queue behind that call.
- disabled = await task_v3_disabled(workflow_run_id, organization_id)
- # billing_tier rides along so a release condition can hold enterprise and self-serve
- # at different percentages. Deliberately not passed to task_v3_disabled: that flag's
- # two callers must build identical provider cache keys, see its docstring.
- if not disabled:
+ # evaluations use the same cache key, wins over the experiment and over the
+ # new-workflow default below. A kill flipped mid-run still takes effect at dispatch
+ # while these rows already read v3: stopping the run wins over its attribution. Read
+ # FIRST, and before the billing lookup below: a Redis or pooler incident is exactly
+ # when someone flips this, and this lock is held throughout, so the kill switch must
+ # not queue behind that call.
+ if await task_v3_disabled(workflow_run_id, organization_id):
+ route_reason = WorkflowBlockEngineRouteReason.disabled
+ else:
+ # billing_tier rides along so a release condition can hold enterprise and
+ # self-serve at different percentages. Deliberately not passed to
+ # task_v3_disabled: that flag's two callers must build identical provider cache
+ # keys, see its docstring.
billing_tier = await _billing_tier_for_arm(organization_id)
- if await provider.is_feature_enabled_cached(
- WORKFLOW_TASK_V3_AB_FLAG,
- workflow_run_id,
- properties={
- "organization_id": organization_id,
- "workflow_permanent_id": workflow_permanent_id or "not_workflow",
- BILLING_TIER_PROPERTY: billing_tier.value,
- },
- ):
+ # The tier, the status and the trigger are checked first, so anything that
+ # cannot use the rule pays for no workflow read. An auto_generated running
+ # version is a per-call workflow the login, download_files, credential
+ # test-login and SDK endpoints mint and run in one request, and enrolling those
+ # would move standing API traffic rather than the workflows a customer keeps;
+ # the birth version's status would be the wrong test, because the prompt box
+ # writes version 1 as auto_generated and the editor save that keeps it writes
+ # the next as published. The job-recipe endpoints are per-call too but build a
+ # published definition, so only their trigger kind excludes them.
+ takes_new_workflow_default = (
+ billing_tier == BillingTier.SELF_SERVE
+ and workflow_status != WorkflowStatus.auto_generated
+ and trigger_type not in PER_CALL_WORKFLOW_RUN_TRIGGER_TYPES
+ and await _workflow_is_new_for_v3_default(workflow_permanent_id, organization_id)
+ )
+ if takes_new_workflow_default:
+ # Read only for the runs the rule would enrol, so the A/B population is never
+ # exposed to this flag and its bucketing is unchanged.
+ rollout = await _resolve_new_workflow_default_rollout(
+ provider,
+ workflow_run_id=workflow_run_id,
+ organization_id=organization_id,
+ workflow_permanent_id=workflow_permanent_id,
+ billing_tier=billing_tier,
+ )
+ if takes_new_workflow_default and _rollout_enrols(rollout):
+ # A workflow born at or after the cutoff runs its task blocks on v3 by
+ # default, so the percentage knob is never consulted: the unenrolled share the
+ # A/B then routes to control is the only concurrent control these runs have.
override = RunEngine.skyvern_v3
+ route_reason = WorkflowBlockEngineRouteReason.new_self_serve_workflow_default
+ else:
+ in_treatment = await _ab_flag_puts_run_in_treatment(
+ provider,
+ workflow_run_id=workflow_run_id,
+ organization_id=organization_id,
+ workflow_permanent_id=workflow_permanent_id,
+ billing_tier=billing_tier,
+ )
+ if in_treatment:
+ override = RunEngine.skyvern_v3
+ route_reason = WorkflowBlockEngineRouteReason.flag_bucket_treatment
+ elif in_treatment is None:
+ # The flag never answered, so this run is on control without having been
+ # randomized. Labelled apart from the bucket so the control cell every
+ # per-arm read builds out of flag_bucket_control stays a randomized one.
+ route_reason = WorkflowBlockEngineRouteReason.flag_undefined
+ else:
+ route_reason = WorkflowBlockEngineRouteReason.flag_bucket_control
except Exception:
LOG.warning(
"Failed to resolve the workflow-block engine arm; using control",
@@ -107,6 +346,7 @@ async def resolve_workflow_block_engine_arm(
exc_info=True,
)
override = None
+ route_reason = WorkflowBlockEngineRouteReason.flag_error
context.workflow_block_engine_override = override
context.workflow_block_engine_resolved_run_id = workflow_run_id
LOG.info(
@@ -114,6 +354,18 @@ async def resolve_workflow_block_engine_arm(
workflow_run_id=workflow_run_id,
workflow_permanent_id=workflow_permanent_id,
arm="treatment" if override else "control",
+ route_reason=route_reason,
+ # True only for a run this rule enrolled, which is every run it treated: the rule's
+ # control cell is the unenrolled share the A/B then routed to control, so a read of the
+ # rule compares route_reason=new_self_serve_workflow_default against
+ # new_workflow_default_rollout_resolution=not_enrolled intersected with
+ # flag_bucket_control -- an unenrolled run re-enters the A/B and can be treated there.
+ new_workflow_default_rollout=_rollout_enrols(rollout),
+ # None whenever the rollout flag was not consulted. The three non-enrolling resolutions
+ # are indistinguishable in the boolean above, and only not_enrolled is a randomized cell,
+ # so both a check of whether the flag is answering at all and the rule's control cell read
+ # this field instead.
+ new_workflow_default_rollout_resolution=rollout,
run_is_eligible=run_is_eligible,
ineligibility_reason=ineligibility_reason,
# None whenever the A/B was never consulted -- an ineligible run, or one the kill switch
diff --git a/skyvern/forge/sdk/workflow/service.py b/skyvern/forge/sdk/workflow/service.py
index e81c070e4..483a46406 100644
--- a/skyvern/forge/sdk/workflow/service.py
+++ b/skyvern/forge/sdk/workflow/service.py
@@ -5827,6 +5827,8 @@ class WorkflowService:
workflow_run_id=workflow_run_id,
organization_id=organization_id,
workflow_permanent_id=workflow_run.workflow_permanent_id,
+ workflow_status=workflow.status,
+ trigger_type=workflow_run.trigger_type,
ineligibility_reason=v3_ab_ineligibility_reason(all_blocks, is_script_run=is_script_run),
)
else:
diff --git a/tests/unit/test_config.py b/tests/unit/test_config.py
index 49047c010..ad8b5a94e 100644
--- a/tests/unit/test_config.py
+++ b/tests/unit/test_config.py
@@ -1,4 +1,5 @@
import logging
+from datetime import UTC, datetime
import pytest
@@ -113,3 +114,28 @@ def test_api_limit_concurrency_rejects_negative(monkeypatch: pytest.MonkeyPatch)
with pytest.raises(ValueError):
Settings(_env_file=None)
+
+
+ ("raw_value", "expected"),
+ [
+ ("2026-09-15T00:00:00Z", datetime(2026, 9, 15, tzinfo=UTC)),
+ ("2026-09-15 00:00:00", datetime(2026, 9, 15)),
+ ("2026-09-15", datetime(2026, 9, 15)),
+ # This setting is the settings-side kill path for the new-workflow v3 default, and blanking
+ # an already-set env var is how it gets turned off. A boot-time ValidationError would
+ # CrashLoopBackOff every arm-resolving process during the incident the operator is ending.
+ ("", None),
+ ("0", None),
+ ("none", None),
+ ("null", None),
+ ("off", None),
+ ("disabled", None),
+ ],
+)
+def test_task_v3_default_engine_workflow_cutoff_env_values(
+ monkeypatch: pytest.MonkeyPatch, raw_value: str, expected: datetime | None
+) -> None:
+ monkeypatch.setenv("TASK_V3_DEFAULT_ENGINE_WORKFLOW_CUTOFF", raw_value)
+
+ assert Settings(_env_file=None).TASK_V3_DEFAULT_ENGINE_WORKFLOW_CUTOFF == expected
diff --git a/tests/unit/test_workflow_block_engine.py b/tests/unit/test_workflow_block_engine.py
index d1df6a4aa..1580a177e 100644
--- a/tests/unit/test_workflow_block_engine.py
+++ b/tests/unit/test_workflow_block_engine.py
@@ -5,22 +5,29 @@ workflow_run_blocks and the dispatched engine come from the same resolution.
from __future__ import annotations
+import time
from collections.abc import Callable, Iterator
-from datetime import UTC, datetime
+from datetime import UTC, datetime, timedelta, timezone
from types import SimpleNamespace
-from typing import Any
+from typing import Any, NamedTuple
from unittest.mock import AsyncMock, MagicMock, patch
import pytest
+from skyvern.config import settings
from skyvern.forge import app
from skyvern.forge.sdk.core import skyvern_context
from skyvern.forge.sdk.core.skyvern_context import SkyvernContext
+from skyvern.forge.sdk.db.enums import WorkflowRunTriggerType
from skyvern.forge.sdk.experimentation.billing_tier import BILLING_TIER_PROPERTY, BillingTier
from skyvern.forge.sdk.experimentation.providers import BaseExperimentationProvider, NoOpExperimentationProvider
from skyvern.forge.sdk.experimentation.workflow_block_engine import (
DISABLE_TASK_V3_FLAG,
+ TASK_V3_NEW_WORKFLOW_DEFAULT_ROLLOUT_FLAG,
WORKFLOW_TASK_V3_AB_FLAG,
+ NewWorkflowDefaultRollout,
+ WorkflowBlockEngineRouteReason,
+ _as_utc,
resolve_workflow_block_engine_arm,
workflow_block_engine_override,
)
@@ -51,7 +58,7 @@ from skyvern.forge.sdk.workflow.models.block import (
)
from skyvern.forge.sdk.workflow.service import WorkflowService
from skyvern.schemas.run_enums import RunEngine
-from skyvern.schemas.workflows import BlockResult, BlockType
+from skyvern.schemas.workflows import BlockResult, BlockType, WorkflowStatus
from skyvern.services import script_service
from tests.unit.helpers import make_organization
from tests.unit.test_agent_task_v3 import (
@@ -67,17 +74,70 @@ WORKFLOW_BLOCK_ENGINE_APP_TARGET = "skyvern.forge.sdk.experimentation.workflow_b
class _FakeExperimentationProvider(BaseExperimentationProvider):
- def __init__(self, flags: dict[str, bool] | None = None, raise_error: bool = False) -> None:
+ """Shaped like the two cloud providers, which is what makes a direction-sensitive flag testable.
+
+ In production an evaluation error never escapes ``_is_feature_enabled``: both PostHog providers
+ swallow it into ``None`` and ``bool()`` it to ``False``, so only ``_resolve_feature_flag_strict``
+ can tell a failure from a real ``False``. A fake that raised from ``_is_feature_enabled`` would
+ verify a shape no provider has, and would let a fail-safe that is inverted in production pass.
+
+ ``False`` here is the answer an INACTIVE flag gives: posthog's local evaluator returns a
+ conclusive ``False`` for one before it reads any filter, so "disable the flag" and "roll it to 0%"
+ arrive as the same value, and a rule that fired on ``False`` would enrol everybody the moment an
+ operator switched the flag off.
+ """
+
+ def __init__(
+ self,
+ flags: dict[str, bool] | None = None,
+ raise_error: bool = False,
+ strict_error_flags: set[str] | None = None,
+ unresolvable_flags: set[str] | None = None,
+ ) -> None:
super().__init__()
self.flags = dict(flags or {})
+ # The rule fires only on a conclusive True, so a fake that did not mention this flag would
+ # answer "undefined" and quietly take every enrolment test in this file off the rule and onto
+ # the A/B path, where control and an unenrolled run look identical. Tests that want the other
+ # resolutions say so, with a False here or with strict_error_flags/unresolvable_flags.
+ self.flags.setdefault(TASK_V3_NEW_WORKFLOW_DEFAULT_ROLLOUT_FLAG, True)
self.calls: list[tuple[str, str, dict | None]] = []
self.raise_error = raise_error
+ self.strict_error_flags = set(strict_error_flags or ())
+ # A flag key local evaluation cannot answer: absent from the snapshot the poller wrote, or
+ # carrying a condition only the PostHog API can resolve. Neither resolver raises; both
+ # return None.
+ self.unresolvable_flags = set(unresolvable_flags or ())
+
+ async def _prepare_feature_flag_resolution(self, feature_name: str, *, cached: bool) -> None:
+ # Where a provider does raise in production: the local provider reloads its flag snapshot
+ # from the database here, and a database failure escapes instead of resolving to a value.
+ if self.raise_error:
+ raise RuntimeError("provider unavailable")
- async def _is_feature_enabled(self, feature_name: str, distinct_id: str, properties: dict | None = None) -> bool:
+ async def _evaluate(self, feature_name: str, distinct_id: str, properties: dict | None) -> bool | None:
self.calls.append((feature_name, distinct_id, properties))
- if self.raise_error:
+ if feature_name in self.strict_error_flags:
raise RuntimeError("provider unavailable")
- return self.flags.get(feature_name, False)
+ if feature_name in self.unresolvable_flags:
+ return None
+ return self.flags.get(feature_name)
+
+ async def _resolve_feature_flag_strict(
+ self, feature_name: str, distinct_id: str, properties: dict | None = None
+ ) -> bool | None:
+ return await self._evaluate(feature_name, distinct_id, properties)
+
+ async def _resolve_feature_flag(
+ self, feature_name: str, distinct_id: str, properties: dict | None = None
+ ) -> bool | None:
+ try:
+ return await self._evaluate(feature_name, distinct_id, properties)
+ except Exception:
+ return None
+
+ async def _is_feature_enabled(self, feature_name: str, distinct_id: str, properties: dict | None = None) -> bool:
+ return bool(await self._resolve_feature_flag(feature_name, distinct_id, properties))
async def _get_value(self, feature_name: str, distinct_id: str, properties: dict | None = None) -> str | None:
return None
@@ -96,6 +156,23 @@ def scoped_context() -> Iterator[SkyvernContext]:
skyvern_context.reset()
+class _Resolution(NamedTuple):
+ log: dict[str, Any]
+ birth_reads: AsyncMock
+ warnings: list[Any]
+
+
+def _consulted_flags(provider: _FakeExperimentationProvider) -> list[str]:
+ return [flag for flag, _distinct_id, _properties in provider.calls]
+
+
+# When the LATEST version of every workflow in this file was saved, deliberately after the
+# v3_default_cutoff fixture below: that models an old workflow edited after the cutoff, so a resolver
+# reading the version this run executes instead of the permanent id's birth flips the arm and is
+# caught behaviorally rather than by an assertion about a mock's call kwargs.
+_LATEST_VERSION_CREATED_AT = datetime(2026, 9, 20)
+
+
async def _resolve(
context: SkyvernContext,
provider: BaseExperimentationProvider,
@@ -104,17 +181,75 @@ async def _resolve(
ineligibility_reason: V3AbIneligibleReason | None,
organization_id: str | None = "org_1",
workflow_permanent_id: str | None = "wpid_1",
-) -> None:
- with patch(WORKFLOW_BLOCK_ENGINE_APP_TARGET) as mock_app:
+ billing_tier: BillingTier = BillingTier.UNKNOWN,
+ first_version_created_at: datetime | None = None,
+ first_version_status: WorkflowStatus = WorkflowStatus.published,
+ first_version_error: Exception | None = None,
+ workflow_status: WorkflowStatus = WorkflowStatus.published,
+ trigger_type: WorkflowRunTriggerType | None = WorkflowRunTriggerType.api,
+) -> _Resolution:
+ async def read_birth_timestamp(workflow_permanent_id: str, organization_id: str) -> datetime | None:
+ if first_version_error is not None:
+ raise first_version_error
+ return first_version_created_at
+
+ async def read_workflow_version(
+ workflow_permanent_id: str, *, version: int | None = None, **_: Any
+ ) -> SimpleNamespace | None:
+ if version != 1:
+ return SimpleNamespace(created_at=_LATEST_VERSION_CREATED_AT, status=WorkflowStatus.published)
+ if first_version_created_at is None:
+ return None
+ return SimpleNamespace(created_at=first_version_created_at, status=first_version_status)
+
+ birth_reads = AsyncMock(side_effect=read_birth_timestamp)
+ with (
+ patch(WORKFLOW_BLOCK_ENGINE_APP_TARGET) as mock_app,
+ patch("skyvern.forge.sdk.experimentation.workflow_block_engine.LOG") as mock_log,
+ ):
mock_app.EXPERIMENTATION_PROVIDER = provider
- stub_workflow_block_engine_app(mock_app)
+ stub_workflow_block_engine_app(mock_app, billing_tier=billing_tier)
+ # spec'd to the one repository the rule may reach, and to the two reads it could plausibly
+ # use: on a bare MagicMock any attribute path answers, so a resolver reading a method the
+ # real AgentDB does not have would pass here and raise in production, where the helper's
+ # catch-all would bury it as "not a new workflow".
+ database = MagicMock(spec=["workflows"])
+ database.workflows = MagicMock(spec=["get_workflow_permanent_id_created_at", "get_workflow_by_permanent_id"])
+ database.workflows.get_workflow_permanent_id_created_at = birth_reads
+ # Answers whatever version is asked for, carrying the birth version's own status at version 1:
+ # a resolver that regressed to deciding the per-call exclusion on the BIRTH version's status
+ # reds the prompt-created case, and one that read the executing version's timestamp reds the
+ # long-lived-workflow case, both behaviorally.
+ database.workflows.get_workflow_by_permanent_id = AsyncMock(side_effect=read_workflow_version)
+ mock_app.DATABASE = database
await resolve_workflow_block_engine_arm(
context,
workflow_run_id=workflow_run_id,
organization_id=organization_id,
workflow_permanent_id=workflow_permanent_id,
+ workflow_status=workflow_status,
+ trigger_type=trigger_type,
ineligibility_reason=ineligibility_reason,
)
+ logged = dict(mock_log.info.call_args.kwargs) if mock_log.info.call_args else {}
+ return _Resolution(log=logged, birth_reads=birth_reads, warnings=list(mock_log.warning.call_args_list))
+
+
[email protected](autouse=True)
+def _no_ambient_v3_default_cutoff(monkeypatch: pytest.MonkeyPatch) -> None:
+ # The rollout this file covers sets TASK_V3_DEFAULT_ENGINE_WORKFLOW_CUTOFF in the environment, and
+ # Settings reads .env at import: without pinning it, every test here answers to whatever the
+ # machine happens to export.
+ monkeypatch.setattr(settings, "TASK_V3_DEFAULT_ENGINE_WORKFLOW_CUTOFF", None)
+
+
+def v3_default_cutoff(monkeypatch: pytest.MonkeyPatch, _no_ambient_v3_default_cutoff: None) -> datetime:
+ # Ordered after the autouse pin above by requesting it, so the off-by-default pin cannot clobber
+ # the cutoff a test asked for.
+ cutoff = datetime(2026, 9, 15, tzinfo=UTC)
+ monkeypatch.setattr(settings, "TASK_V3_DEFAULT_ENGINE_WORKFLOW_CUTOFF", cutoff)
+ return cutoff
def test_mixed_eligibility_run_download_block_is_eligible(scoped_context: SkyvernContext) -> None:
@@ -309,11 +444,48 @@ async def test_disable_flag_wins_over_ab_flag(scoped_context: SkyvernContext) ->
@pytest.mark.asyncio
async def test_provider_exception_fails_closed_to_control(scoped_context: SkyvernContext) -> None:
provider = _FakeExperimentationProvider(raise_error=True)
- await _resolve(scoped_context, provider, workflow_run_id="wr_err", ineligibility_reason=None)
+ resolution = await _resolve(scoped_context, provider, workflow_run_id="wr_err", ineligibility_reason=None)
assert scoped_context.workflow_block_engine_override is None
block = _make_block(TaskBlock, label="err_block")
assert block.resolve_engine("wr_err") == RunEngine.skyvern_v1
+ # An unrandomized control run has to be separable from a bucketed one in the same read.
+ assert resolution.log["route_reason"] == WorkflowBlockEngineRouteReason.flag_error
+
+
+async def test_a_failing_ab_evaluation_is_labelled_flag_error_not_bucketed_control(
+ scoped_context: SkyvernContext,
+) -> None:
+ # A failed A/B evaluation and a real control bucket produce the same arm, so only route_reason
+ # separates them. Reading a provider outage as flag_bucket_control mixes runs that were never
+ # randomized into the control cell of every per-arm comparison, which is the one thing this field
+ # exists to prevent. Only reachable through the strict resolver: the boolean one swallows the
+ # error into None and bool()s it to a plain False.
+ provider = _FakeExperimentationProvider(strict_error_flags={WORKFLOW_TASK_V3_AB_FLAG})
+ resolution = await _resolve(scoped_context, provider, workflow_run_id="wr_ab_down", ineligibility_reason=None)
+
+ assert scoped_context.workflow_block_engine_override is None
+ assert _make_block(TaskBlock, label="ab_down").resolve_engine("wr_ab_down") == RunEngine.skyvern_v1
+ assert resolution.log["route_reason"] == WorkflowBlockEngineRouteReason.flag_error
+
+
+async def test_an_ab_flag_local_evaluation_cannot_answer_is_control_but_not_a_bucket(
+ scoped_context: SkyvernContext,
+) -> None:
+ # The third A/B answer: None from local evaluation is neither a failure nor a bucket. A key the
+ # snapshot has no row for, or a condition local evaluation cannot answer, leaves the run on
+ # control without it ever having been randomized -- and the runbook builds this rule's control
+ # cell out of flag_bucket_control, so labelling it that way pours never-randomized runs into
+ # every per-arm comparison, the same defect flag_error exists to prevent. The arm is unchanged.
+ provider = _FakeExperimentationProvider(
+ {WORKFLOW_TASK_V3_AB_FLAG: True}, unresolvable_flags={WORKFLOW_TASK_V3_AB_FLAG}
+ )
+ resolution = await _resolve(scoped_context, provider, workflow_run_id="wr_ab_undefined", ineligibility_reason=None)
+
+ assert scoped_context.workflow_block_engine_override is None
+ assert resolution.log["route_reason"] == WorkflowBlockEngineRouteReason.flag_undefined
def test_script_run_is_never_eligible() -> None:
@@ -336,6 +508,8 @@ async def test_noop_provider_never_queried_and_leaves_engine_unchanged(scoped_co
workflow_run_id="wr_noop",
organization_id="org_1",
workflow_permanent_id="wpid_1",
+ workflow_status=WorkflowStatus.published,
+ trigger_type=WorkflowRunTriggerType.api,
ineligibility_reason=None,
)
@@ -465,25 +639,35 @@ async def test_base_task_block_execute_dispatches_with_the_resolved_engine(scope
assert captured["engine"] == RunEngine.skyvern_v3
-async def test_execute_workflow_blocks_pins_the_context_execute_safe_reads_from(
- scoped_context: SkyvernContext,
+async def _persisted_engine_from_execute_workflow_blocks(
monkeypatch: pytest.MonkeyPatch,
-) -> None:
+ provider: BaseExperimentationProvider,
+ *,
+ workflow_status: WorkflowStatus = WorkflowStatus.published,
+ billing_tier: BillingTier = BillingTier.UNKNOWN,
+ first_version_created_at: datetime | None = None,
+ trigger_type: WorkflowRunTriggerType | None = WorkflowRunTriggerType.api,
+) -> RunEngine:
"""Drives the real WorkflowService._execute_workflow_blocks -> _execute_single_block ->
- Block.execute_safe chain for a single eligible TaskBlock, forcing only the experimentation
- provider. If resolve_workflow_block_engine_arm pinned the arm on a different SkyvernContext
- object than the one execute_safe's resolve_engine() reads back from, this fails: the
- persisted engine would be skyvern_v1 instead of skyvern_v3.
+ Block.execute_safe chain for a single eligible TaskBlock and returns the engine it persisted,
+ forcing only the experimentation provider, the billing tier and the workflow's birth timestamp.
+ Everything the arm decision depends on is therefore supplied by the real caller, not by the
+ resolver's own test seam.
"""
- provider = _FakeExperimentationProvider({WORKFLOW_TASK_V3_AB_FLAG: True})
monkeypatch.setattr(app, "EXPERIMENTATION_PROVIDER", provider)
+ monkeypatch.setattr(app.AGENT_FUNCTION, "resolve_billing_tier", AsyncMock(return_value=billing_tier))
+ monkeypatch.setattr(
+ app.DATABASE.workflows,
+ "get_workflow_permanent_id_created_at",
+ AsyncMock(return_value=first_version_created_at),
+ )
block = _make_block(TaskBlock, label="e2e_block")
workflow = MagicMock()
workflow.workflow_definition.blocks = [block]
workflow.workflow_definition.version = 1
workflow.workflow_definition.finally_block_label = None
+ workflow.status = workflow_status
workflow_run = MagicMock()
workflow_run.workflow_run_id = "wr_e2e"
@@ -491,6 +675,7 @@ async def test_execute_workflow_blocks_pins_the_context_execute_safe_reads_from(
workflow_run.organization_id = "org_e2e"
workflow_run.run_with = None
workflow_run.retried_from_workflow_run_id = None
+ workflow_run.trigger_type = trigger_type
organization = MagicMock()
organization.organization_id = "org_e2e"
@@ -527,7 +712,91 @@ async def test_execute_workflow_blocks_pins_the_context_execute_safe_reads_from(
organization=organization,
)
- assert captured["engine"] == RunEngine.skyvern_v3
+ return captured["engine"]
+
+
+async def test_execute_workflow_blocks_pins_the_context_execute_safe_reads_from(
+ scoped_context: SkyvernContext,
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ """If resolve_workflow_block_engine_arm pinned the arm on a different SkyvernContext object
+ than the one execute_safe's resolve_engine() reads back from, this fails: the persisted engine
+ would be skyvern_v1 instead of skyvern_v3.
+ """
+ provider = _FakeExperimentationProvider({WORKFLOW_TASK_V3_AB_FLAG: True})
+
+ engine = await _persisted_engine_from_execute_workflow_blocks(monkeypatch, provider)
+
+ assert engine == RunEngine.skyvern_v3
+
+
+ ("workflow_status", "expected_engine"),
+ [
+ (WorkflowStatus.auto_generated, RunEngine.skyvern_v1),
+ (WorkflowStatus.published, RunEngine.skyvern_v3),
+ ],
+)
+async def test_execute_workflow_blocks_hands_the_resolver_the_running_versions_status(
+ scoped_context: SkyvernContext,
+ monkeypatch: pytest.MonkeyPatch,
+ v3_default_cutoff: datetime,
+ workflow_status: WorkflowStatus,
+ expected_engine: RunEngine,
+) -> None:
+ # The per-call exclusion is only as real as the status the caller passes. Both cases are the same
+ # new self-serve workflow with the percentage knob off, so the only thing deciding the engine is
+ # workflow.status reaching the resolver: a caller that stopped passing it, or passed the
+ # workflow_run instead, would take the login/download/credential-test/SDK endpoints' per-call
+ # workflows onto v3 with it.
+ provider = _FakeExperimentationProvider({WORKFLOW_TASK_V3_AB_FLAG: False})
+
+ engine = await _persisted_engine_from_execute_workflow_blocks(
+ monkeypatch,
+ provider,
+ workflow_status=workflow_status,
+ billing_tier=BillingTier.SELF_SERVE,
+ first_version_created_at=v3_default_cutoff.replace(tzinfo=None) + timedelta(days=1),
+ )
+
+ assert engine == expected_engine
+
+
+ ("trigger_type", "expected_engine"),
+ [
+ (WorkflowRunTriggerType.job_recipe_apply, RunEngine.skyvern_v1),
+ (WorkflowRunTriggerType.job_recipe_extract, RunEngine.skyvern_v1),
+ (WorkflowRunTriggerType.api, RunEngine.skyvern_v3),
+ ],
+)
+async def test_execute_workflow_blocks_hands_the_resolver_the_runs_trigger_type(
+ scoped_context: SkyvernContext,
+ monkeypatch: pytest.MonkeyPatch,
+ v3_default_cutoff: datetime,
+ trigger_type: WorkflowRunTriggerType,
+ expected_engine: RunEngine,
+) -> None:
+ # The job-recipe endpoints run a brand-new published workflow_permanent_id per request, so the
+ # status the caller passes cannot exclude them and only the run's trigger kind can. All three
+ # cases are the same new self-serve published workflow with the percentage knob off, so the only
+ # thing deciding the engine is workflow_run.trigger_type reaching the resolver: a caller that
+ # stopped passing it would force-route every recipe call to v3, unrandomized.
+ provider = _FakeExperimentationProvider({WORKFLOW_TASK_V3_AB_FLAG: False})
+
+ engine = await _persisted_engine_from_execute_workflow_blocks(
+ monkeypatch,
+ provider,
+ workflow_status=WorkflowStatus.published,
+ billing_tier=BillingTier.SELF_SERVE,
+ first_version_created_at=v3_default_cutoff.replace(tzinfo=None) + timedelta(days=1),
+ trigger_type=trigger_type,
+ )
+
+ assert engine == expected_engine
def _eligible_mix_blocks() -> list[BaseTaskBlock]:
@@ -668,6 +937,8 @@ async def test_resolver_logs_the_ineligibility_reason(scoped_context: SkyvernCon
workflow_run_id="wr_logged",
organization_id="org_1",
workflow_permanent_id="wpid_1",
+ workflow_status=WorkflowStatus.published,
+ trigger_type=WorkflowRunTriggerType.api,
ineligibility_reason=V3AbIneligibleReason.pinned_engine,
)
@@ -834,6 +1105,8 @@ async def test_arm_offers_the_billing_tier_to_the_ab_flag_but_not_the_kill_switc
workflow_run_id="wr_tier",
organization_id="org_1",
workflow_permanent_id="wpid_1",
+ workflow_status=WorkflowStatus.published,
+ trigger_type=WorkflowRunTriggerType.api,
ineligibility_reason=None,
)
@@ -864,6 +1137,8 @@ async def test_a_run_that_never_reaches_the_ab_pays_for_no_tier_lookup(scoped_co
workflow_run_id=run_id,
organization_id="org_1",
workflow_permanent_id="wpid_1",
+ workflow_status=WorkflowStatus.published,
+ trigger_type=WorkflowRunTriggerType.api,
ineligibility_reason=reason,
)
tier.assert_not_awaited()
@@ -885,9 +1160,474 @@ async def test_a_failing_billing_tier_lookup_cannot_decide_the_arm(scoped_contex
workflow_run_id="wr_tier_down",
organization_id="org_1",
workflow_permanent_id="wpid_1",
+ workflow_status=WorkflowStatus.published,
+ trigger_type=WorkflowRunTriggerType.api,
ineligibility_reason=None,
)
assert scoped_context.workflow_block_engine_override == RunEngine.skyvern_v3
by_flag = {flag: properties or {} for flag, _distinct_id, properties in provider.calls}
assert by_flag[WORKFLOW_TASK_V3_AB_FLAG][BILLING_TIER_PROPERTY] == "unknown"
+
+
+async def test_new_self_serve_workflow_defaults_to_v3_without_consulting_the_ab_flag(
+ scoped_context: SkyvernContext,
+ v3_default_cutoff: datetime,
+) -> None:
+ # The whole point of the rule: the percentage knob is bypassed, so a flag sitting at 0% (or a
+ # bucket that landed on control) cannot take a new self-serve workflow back to v1.
+ provider = _FakeExperimentationProvider(
+ {WORKFLOW_TASK_V3_AB_FLAG: False, TASK_V3_NEW_WORKFLOW_DEFAULT_ROLLOUT_FLAG: True}
+ )
+ run_id = "wr_new_self_serve_enrolled"
+ resolution = await _resolve(
+ scoped_context,
+ provider,
+ workflow_run_id=run_id,
+ ineligibility_reason=None,
+ billing_tier=BillingTier.SELF_SERVE,
+ first_version_created_at=v3_default_cutoff.replace(tzinfo=None) + timedelta(seconds=1),
+ )
+
+ assert scoped_context.workflow_block_engine_override == RunEngine.skyvern_v3
+ assert _make_block(TaskBlock, label="new_wf").resolve_engine(run_id) == RunEngine.skyvern_v3
+ # The percentage knob is still never consulted for an enrolled run.
+ assert _consulted_flags(provider) == [DISABLE_TASK_V3_FLAG, TASK_V3_NEW_WORKFLOW_DEFAULT_ROLLOUT_FLAG]
+ assert resolution.log["route_reason"] == WorkflowBlockEngineRouteReason.new_self_serve_workflow_default
+ assert resolution.log["new_workflow_default_rollout"] is True
+ assert resolution.log["new_workflow_default_rollout_resolution"] == NewWorkflowDefaultRollout.enrolled
+ assert resolution.log["billing_tier"] == BillingTier.SELF_SERVE.value
+
+
+ ("ab_flag", "expected_override", "expected_reason"),
+ [
+ (True, RunEngine.skyvern_v3, WorkflowBlockEngineRouteReason.flag_bucket_treatment),
+ (False, None, WorkflowBlockEngineRouteReason.flag_bucket_control),
+ ],
+)
+async def test_an_unenrolled_new_workflow_is_bucketed_by_the_ab_as_if_the_rule_did_not_exist(
+ scoped_context: SkyvernContext,
+ v3_default_cutoff: datetime,
+ ab_flag: bool,
+ expected_override: RunEngine | None,
+ expected_reason: WorkflowBlockEngineRouteReason,
+) -> None:
+ # A conclusive False is what an operator's fastest off-gesture produces: posthog's local evaluator
+ # answers False for an INACTIVE flag before it reads any filter, and the same value comes back for
+ # a run outside the percentage or excluded by a condition. Enrolling on it would put 100% of this
+ # population on v3 with the control cell zeroed at the exact moment someone switched the rollout
+ # off, and the log would read like a rule that was not firing. So False leaves the run in the
+ # ordinary bucketing, both ways -- which is also where this population's concurrent control and
+ # its scoped kill come from, the cutoff being a setting that needs a restart and a condition on
+ # WORKFLOW_TASK_V3_AB being unable to reach a run that never evaluates it.
+ provider = _FakeExperimentationProvider(
+ {WORKFLOW_TASK_V3_AB_FLAG: ab_flag, TASK_V3_NEW_WORKFLOW_DEFAULT_ROLLOUT_FLAG: False}
+ )
+ resolution = await _resolve(
+ scoped_context,
+ provider,
+ workflow_run_id=f"wr_not_enrolled_{ab_flag}",
+ ineligibility_reason=None,
+ billing_tier=BillingTier.SELF_SERVE,
+ first_version_created_at=v3_default_cutoff.replace(tzinfo=None) + timedelta(days=1),
+ )
+
+ assert scoped_context.workflow_block_engine_override == expected_override
+ assert resolution.log["route_reason"] == expected_reason
+ assert resolution.log["new_workflow_default_rollout"] is False
+ assert resolution.log["new_workflow_default_rollout_resolution"] == NewWorkflowDefaultRollout.not_enrolled
+ rollout_call = next(call for call in provider.calls if call[0] == TASK_V3_NEW_WORKFLOW_DEFAULT_ROLLOUT_FLAG)
+ # Bucketed per run, and carrying the two properties a scoped condition is written against: holding
+ # one organization or one workflow out of the rollout is what the runbook promises as the fast
+ # lever.
+ assert rollout_call[1] == f"wr_not_enrolled_{ab_flag}"
+ assert rollout_call[2] == {
+ "organization_id": "org_1",
+ "workflow_permanent_id": "wpid_1",
+ BILLING_TIER_PROPERTY: BillingTier.SELF_SERVE.value,
+ }
+
+
+async def test_a_failing_rollout_evaluation_leaves_the_run_on_the_ab_path(
+ scoped_context: SkyvernContext, v3_default_cutoff: datetime
+) -> None:
+ # An evaluation that raised is not an answer, and treating it as one would enrol a population
+ # whose fastest kill is the flag that just failed. It also must not be reported as a real
+ # "outside the percentage": the error resolution is what separates the two on the log. Only
+ # reachable through the strict resolver -- the boolean one swallows the error into None and
+ # bool()s it to False.
+ provider = _FakeExperimentationProvider(
+ {WORKFLOW_TASK_V3_AB_FLAG: False}, strict_error_flags={TASK_V3_NEW_WORKFLOW_DEFAULT_ROLLOUT_FLAG}
+ )
+ resolution = await _resolve(
+ scoped_context,
+ provider,
+ workflow_run_id="wr_rollout_down",
+ ineligibility_reason=None,
+ billing_tier=BillingTier.SELF_SERVE,
+ first_version_created_at=v3_default_cutoff.replace(tzinfo=None) + timedelta(days=1),
+ )
+
+ assert scoped_context.workflow_block_engine_override is None
+ assert resolution.log["route_reason"] == WorkflowBlockEngineRouteReason.flag_bucket_control
+ assert resolution.log["new_workflow_default_rollout"] is False
+ assert resolution.log["new_workflow_default_rollout_resolution"] == NewWorkflowDefaultRollout.error
+ assert WORKFLOW_TASK_V3_AB_FLAG in _consulted_flags(provider)
+ # A rollout that stopped answering must not be silent: an operator reading a quiet
+ # new_self_serve_workflow_default needs this warning and the resolution field above to tell a
+ # broken evaluation from a rollout nobody enabled.
+ assert [call.kwargs.get("workflow_run_id") for call in resolution.warnings] == ["wr_rollout_down"]
+
+
+ ("ab_flag", "expected_override", "expected_reason"),
+ [
+ (True, RunEngine.skyvern_v3, WorkflowBlockEngineRouteReason.flag_bucket_treatment),
+ (False, None, WorkflowBlockEngineRouteReason.flag_bucket_control),
+ ],
+)
+async def test_an_unresolvable_rollout_flag_leaves_the_run_on_the_ab_path(
+ scoped_context: SkyvernContext,
+ v3_default_cutoff: datetime,
+ ab_flag: bool,
+ expected_override: RunEngine | None,
+ expected_reason: WorkflowBlockEngineRouteReason,
+) -> None:
+ # The arm-resolving processes evaluate PostHog locally against a snapshot, which answers None --
+ # not False, not an error -- for a flag key it has no row for; so does a cutoff set before anyone
+ # created the flag, and so does a condition local evaluation cannot answer. Enrolling on that
+ # would make the cutoff alone a 100%-v3 cohort with no concurrent control, and would bypass the
+ # organization exclusions this rollout's own conditions carry. So an unresolved flag leaves the
+ # run in the ordinary bucketing, both ways -- and the resolution field still separates "no flag"
+ # from a rollout that answered, which is what an operator reads after an enable.
+ provider = _FakeExperimentationProvider(
+ {WORKFLOW_TASK_V3_AB_FLAG: ab_flag},
+ unresolvable_flags={TASK_V3_NEW_WORKFLOW_DEFAULT_ROLLOUT_FLAG},
+ )
+ resolution = await _resolve(
+ scoped_context,
+ provider,
+ workflow_run_id=f"wr_rollout_undefined_{ab_flag}",
+ ineligibility_reason=None,
+ billing_tier=BillingTier.SELF_SERVE,
+ first_version_created_at=v3_default_cutoff.replace(tzinfo=None) + timedelta(days=1),
+ )
+
+ assert scoped_context.workflow_block_engine_override == expected_override
+ assert resolution.log["route_reason"] == expected_reason
+ assert resolution.log["new_workflow_default_rollout"] is False
+ assert resolution.log["new_workflow_default_rollout_resolution"] == NewWorkflowDefaultRollout.undefined
+ assert WORKFLOW_TASK_V3_AB_FLAG in _consulted_flags(provider)
+
+
+async def test_a_new_version_of_an_older_workflow_is_not_a_new_workflow(
+ scoped_context: SkyvernContext, v3_default_cutoff: datetime
+) -> None:
+ # A workflow saved again after the cutoff keeps an old birthday. _resolve wires the latest
+ # version at _LATEST_VERSION_CREATED_AT, on the far side of the cutoff, so a resolver that read
+ # the version this run executes would enrol this workflow and flip the arm below to treatment --
+ # which is the regression that would otherwise take every long-lived workflow onto v3 at once.
+ provider = _FakeExperimentationProvider({WORKFLOW_TASK_V3_AB_FLAG: False})
+ resolution = await _resolve(
+ scoped_context,
+ provider,
+ workflow_run_id="wr_old_wf",
+ ineligibility_reason=None,
+ billing_tier=BillingTier.SELF_SERVE,
+ first_version_created_at=v3_default_cutoff.replace(tzinfo=None) - timedelta(days=30),
+ )
+
+ assert scoped_context.workflow_block_engine_override is None
+ assert DISABLE_TASK_V3_FLAG in [flag for flag, _distinct_id, _properties in provider.calls]
+ assert WORKFLOW_TASK_V3_AB_FLAG in [flag for flag, _distinct_id, _properties in provider.calls]
+ assert resolution.log["route_reason"] == WorkflowBlockEngineRouteReason.flag_bucket_control
+ assert TASK_V3_NEW_WORKFLOW_DEFAULT_ROLLOUT_FLAG not in _consulted_flags(provider)
+ # Asserted here rather than inside the fake, whose AssertionError the rule's catch-all would
+ # swallow into "not a new workflow": the birth timestamp is read once, scoped to this run's own
+ # organization. The read itself is the min over every version of the id including deleted ones,
+ # so a deleted or copilot-test first version cannot make a long-lived workflow look unborn.
+ resolution.birth_reads.assert_awaited_once_with("wpid_1", "org_1")
+
+
+async def test_a_per_call_auto_generated_workflow_is_never_a_new_workflow(
+ scoped_context: SkyvernContext, v3_default_cutoff: datetime
+) -> None:
+ # /v1/run/tasks/login, /v1/run/tasks/download_files, the two credential test-login endpoints and
+ # the SDK run endpoint each mint a fresh workflow_permanent_id per call and run it while it is
+ # still auto_generated, holding a v3-eligible block. Every one of those is born after any cutoff,
+ # so without this exclusion setting the cutoff moves all of that standing API traffic onto v3 at
+ # once instead of only the workflows customers keep.
+ provider = _FakeExperimentationProvider({WORKFLOW_TASK_V3_AB_FLAG: False})
+ resolution = await _resolve(
+ scoped_context,
+ provider,
+ workflow_run_id="wr_auto_generated",
+ ineligibility_reason=None,
+ billing_tier=BillingTier.SELF_SERVE,
+ first_version_created_at=v3_default_cutoff.replace(tzinfo=None) + timedelta(days=1),
+ workflow_status=WorkflowStatus.auto_generated,
+ )
+
+ assert scoped_context.workflow_block_engine_override is None
+ assert WORKFLOW_TASK_V3_AB_FLAG in _consulted_flags(provider)
+ assert TASK_V3_NEW_WORKFLOW_DEFAULT_ROLLOUT_FLAG not in _consulted_flags(provider)
+ assert resolution.log["route_reason"] == WorkflowBlockEngineRouteReason.flag_bucket_control
+ assert resolution.log["new_workflow_default_rollout_resolution"] is None
+ # A status the rule cannot use must not pay for the workflow read either.
+ resolution.birth_reads.assert_not_awaited()
+
+
+ "trigger_type",
+ [WorkflowRunTriggerType.job_recipe_apply, WorkflowRunTriggerType.job_recipe_extract],
+)
+async def test_a_per_call_recipe_run_is_never_a_new_workflow(
+ scoped_context: SkyvernContext,
+ v3_default_cutoff: datetime,
+ trigger_type: WorkflowRunTriggerType,
+) -> None:
+ # The job-recipe endpoints also mint a fresh workflow_permanent_id per request and run it in that
+ # same request, but the definition they build carries no status and so is born published: the
+ # auto_generated exclusion cannot see it, and every one of those ids is born after any cutoff.
+ # One run per permanent id also means an enrolled recipe call has no control run of the same
+ # workflow to be matched against, so the trigger kind is what keeps this standing API traffic
+ # randomized by the A/B instead of force-routed.
+ provider = _FakeExperimentationProvider({WORKFLOW_TASK_V3_AB_FLAG: False})
+ resolution = await _resolve(
+ scoped_context,
+ provider,
+ workflow_run_id=f"wr_{trigger_type.value}",
+ ineligibility_reason=None,
+ billing_tier=BillingTier.SELF_SERVE,
+ first_version_created_at=v3_default_cutoff.replace(tzinfo=None) + timedelta(days=1),
+ workflow_status=WorkflowStatus.published,
+ trigger_type=trigger_type,
+ )
+
+ assert scoped_context.workflow_block_engine_override is None
+ assert WORKFLOW_TASK_V3_AB_FLAG in _consulted_flags(provider)
+ assert TASK_V3_NEW_WORKFLOW_DEFAULT_ROLLOUT_FLAG not in _consulted_flags(provider)
+ assert resolution.log["route_reason"] == WorkflowBlockEngineRouteReason.flag_bucket_control
+ assert resolution.log["new_workflow_default_rollout_resolution"] is None
+ # A trigger the rule cannot use must not pay for the workflow read either.
+ resolution.birth_reads.assert_not_awaited()
+
+
+async def test_a_workflow_born_auto_generated_and_kept_by_the_customer_is_a_new_workflow(
+ scoped_context: SkyvernContext, v3_default_cutoff: datetime
+) -> None:
+ # POST /workflows/create-from-prompt -- the prompt box, the product's main new-workflow path --
+ # writes version 1 as auto_generated whenever publish_workflow is false, which is its default, and
+ # then drops the customer into the editor on that same workflow_permanent_id; an editor save
+ # writes the next version as published. Excluding on the BIRTH version's status would therefore
+ # exclude every prompt-created workflow forever, including the ones a customer edits and re-runs
+ # for months, and the rule's whole stated purpose would be defeated for the path that creates most
+ # of its population. The exclusion is about the version this run executes, not about how the id
+ # was born.
+ provider = _FakeExperimentationProvider({WORKFLOW_TASK_V3_AB_FLAG: False})
+ resolution = await _resolve(
+ scoped_context,
+ provider,
+ workflow_run_id="wr_prompt_created_kept",
+ ineligibility_reason=None,
+ billing_tier=BillingTier.SELF_SERVE,
+ first_version_created_at=v3_default_cutoff.replace(tzinfo=None) + timedelta(days=1),
+ first_version_status=WorkflowStatus.auto_generated,
+ workflow_status=WorkflowStatus.published,
+ )
+
+ assert scoped_context.workflow_block_engine_override == RunEngine.skyvern_v3
+ assert resolution.log["route_reason"] == WorkflowBlockEngineRouteReason.new_self_serve_workflow_default
+ assert WORKFLOW_TASK_V3_AB_FLAG not in _consulted_flags(provider)
+
+
+ ("first_version_created_at", "expect_v3_default"),
+ [
+ (datetime(2026, 9, 14, 23, 59, 59), False),
+ # Naive rows come from datetime.utcnow(): the cutoff is aware, so a resolver that compared
+ # them directly would raise and lose every enrolment. At-or-after includes the instant itself.
+ (datetime(2026, 9, 15, 0, 0, 0), True),
+ (datetime(2026, 9, 15, 0, 0, 1), True),
+ (datetime(2026, 9, 15, 0, 0, 1, tzinfo=UTC), True),
+ # 2026-09-14 21:00 at UTC-4 is 2026-09-15 01:00 UTC, i.e. after the cutoff despite the date.
+ (datetime(2026, 9, 14, 21, 0, 0, tzinfo=timezone(timedelta(hours=-4))), True),
+ ],
+)
+async def test_cutoff_compares_aware_and_naive_first_version_timestamps_in_utc(
+ scoped_context: SkyvernContext,
+ v3_default_cutoff: datetime,
+ first_version_created_at: datetime,
+ expect_v3_default: bool,
+) -> None:
+ provider = _FakeExperimentationProvider({WORKFLOW_TASK_V3_AB_FLAG: False})
+ resolution = await _resolve(
+ scoped_context,
+ provider,
+ workflow_run_id=f"wr_cutoff_{first_version_created_at.isoformat()}",
+ ineligibility_reason=None,
+ billing_tier=BillingTier.SELF_SERVE,
+ first_version_created_at=first_version_created_at,
+ )
+
+ expected_override = RunEngine.skyvern_v3 if expect_v3_default else None
+ assert scoped_context.workflow_block_engine_override == expected_override
+ expected_reason = (
+ WorkflowBlockEngineRouteReason.new_self_serve_workflow_default
+ if expect_v3_default
+ else WorkflowBlockEngineRouteReason.flag_bucket_control
+ )
+ assert resolution.log["route_reason"] == expected_reason
+
+
[email protected](not hasattr(time, "tzset"), reason="TZ cannot be changed in-process on this platform")
+def test_a_naive_created_at_is_read_as_utc_not_as_local_time(monkeypatch: pytest.MonkeyPatch) -> None:
+ # "Naive means UTC" and the classic astimezone() spelling that means "naive means local" are the
+ # same function on a UTC machine, which CI and every container is -- so neither the cases above
+ # nor any assertion made under the ambient zone can tell them apart. workflows.created_at is
+ # datetime.utcnow(), so reading it as local time misclassifies every workflow born within the
+ # process's TZ offset of the cutoff, silently enrolling or excluding runs that have no control arm.
+ monkeypatch.setenv("TZ", "America/New_York")
+ time.tzset()
+ try:
+ assert _as_utc(datetime(2026, 9, 15, 12)) == datetime(2026, 9, 15, 12, tzinfo=UTC)
+ finally:
+ monkeypatch.undo()
+ time.tzset()
+
+
[email protected]("billing_tier", [BillingTier.ENTERPRISE, BillingTier.UNKNOWN])
+async def test_only_the_self_serve_tier_gets_the_new_workflow_default(
+ scoped_context: SkyvernContext, v3_default_cutoff: datetime, billing_tier: BillingTier
+) -> None:
+ # UNKNOWN is a failed lookup, so it must fall through with enterprise rather than be force-routed.
+ provider = _FakeExperimentationProvider({WORKFLOW_TASK_V3_AB_FLAG: True})
+ resolution = await _resolve(
+ scoped_context,
+ provider,
+ workflow_run_id=f"wr_tier_{billing_tier.value}",
+ ineligibility_reason=None,
+ billing_tier=billing_tier,
+ first_version_created_at=v3_default_cutoff.replace(tzinfo=None) + timedelta(days=1),
+ )
+
+ assert WORKFLOW_TASK_V3_AB_FLAG in [flag for flag, _distinct_id, _properties in provider.calls]
+ assert resolution.log["route_reason"] == WorkflowBlockEngineRouteReason.flag_bucket_treatment
+ # A tier the rule cannot use must not pay for the workflow read either.
+ resolution.birth_reads.assert_not_awaited()
+ assert TASK_V3_NEW_WORKFLOW_DEFAULT_ROLLOUT_FLAG not in _consulted_flags(provider)
+
+
+async def test_kill_switch_wins_over_the_new_workflow_default(
+ scoped_context: SkyvernContext, v3_default_cutoff: datetime
+) -> None:
+ provider = _FakeExperimentationProvider({WORKFLOW_TASK_V3_AB_FLAG: True, DISABLE_TASK_V3_FLAG: True})
+ resolution = await _resolve(
+ scoped_context,
+ provider,
+ workflow_run_id="wr_new_but_killed",
+ ineligibility_reason=None,
+ billing_tier=BillingTier.SELF_SERVE,
+ first_version_created_at=v3_default_cutoff.replace(tzinfo=None) + timedelta(days=1),
+ )
+
+ assert scoped_context.workflow_block_engine_override is None
+ assert resolution.log["route_reason"] == WorkflowBlockEngineRouteReason.disabled
+ resolution.birth_reads.assert_not_awaited()
+ assert TASK_V3_NEW_WORKFLOW_DEFAULT_ROLLOUT_FLAG not in _consulted_flags(provider)
+
+
+async def test_no_cutoff_leaves_every_workflow_on_the_flag_path(scoped_context: SkyvernContext) -> None:
+ # Deliberately does not request v3_default_cutoff: the autouse pin holds the setting at None.
+ provider = _FakeExperimentationProvider({WORKFLOW_TASK_V3_AB_FLAG: False})
+ resolution = await _resolve(
+ scoped_context,
+ provider,
+ workflow_run_id="wr_no_cutoff",
+ ineligibility_reason=None,
+ billing_tier=BillingTier.SELF_SERVE,
+ first_version_created_at=datetime(2030, 1, 1),
+ )
+
+ assert scoped_context.workflow_block_engine_override is None
+ assert resolution.log["route_reason"] == WorkflowBlockEngineRouteReason.flag_bucket_control
+ resolution.birth_reads.assert_not_awaited()
+ assert TASK_V3_NEW_WORKFLOW_DEFAULT_ROLLOUT_FLAG not in _consulted_flags(provider)
+
+
+async def test_ineligible_run_is_never_defaulted_to_v3(
+ scoped_context: SkyvernContext, v3_default_cutoff: datetime
+) -> None:
+ # Run-level eligibility is about whether the run can be rerouted at all, so the rule changes the
+ # arm decision only: a script run or a pinned block still executes as authored.
+ provider = _FakeExperimentationProvider({WORKFLOW_TASK_V3_AB_FLAG: True})
+ resolution = await _resolve(
+ scoped_context,
+ provider,
+ workflow_run_id="wr_new_but_ineligible",
+ ineligibility_reason=V3AbIneligibleReason.script_run,
+ billing_tier=BillingTier.SELF_SERVE,
+ first_version_created_at=v3_default_cutoff.replace(tzinfo=None) + timedelta(days=1),
+ )
+
+ assert scoped_context.workflow_block_engine_override is None
+ assert provider.calls == []
+ assert resolution.log["route_reason"] == WorkflowBlockEngineRouteReason.ineligible
+ resolution.birth_reads.assert_not_awaited()
+ assert TASK_V3_NEW_WORKFLOW_DEFAULT_ROLLOUT_FLAG not in _consulted_flags(provider)
+
+
+async def test_a_failing_first_version_read_falls_back_to_the_flag_path(
+ scoped_context: SkyvernContext, v3_default_cutoff: datetime
+) -> None:
+ # The new rule must not be able to take the run off the arm it would have had: a database blip
+ # falls back to the A/B, not to control, and never forces v3.
+ provider = _FakeExperimentationProvider({WORKFLOW_TASK_V3_AB_FLAG: True})
+ resolution = await _resolve(
+ scoped_context,
+ provider,
+ workflow_run_id="wr_first_version_down",
+ ineligibility_reason=None,
+ billing_tier=BillingTier.SELF_SERVE,
+ first_version_error=RuntimeError("workflows read failed"),
+ )
+
+ assert scoped_context.workflow_block_engine_override == RunEngine.skyvern_v3
+ assert resolution.log["route_reason"] == WorkflowBlockEngineRouteReason.flag_bucket_treatment
+ # The fallback is silent in the arm it produces, so the swallowed read has to be visible
+ # somewhere: a rule that stops enrolling because a query broke must not look like a rule nobody
+ # set the cutoff for.
+ assert [call.kwargs.get("workflow_permanent_id") for call in resolution.warnings] == ["wpid_1"]
+
+
+async def test_a_missing_first_version_falls_back_to_the_flag_path(
+ scoped_context: SkyvernContext, v3_default_cutoff: datetime
+) -> None:
+ provider = _FakeExperimentationProvider({WORKFLOW_TASK_V3_AB_FLAG: False})
+ resolution = await _resolve(
+ scoped_context,
+ provider,
+ workflow_run_id="wr_first_version_missing",
+ ineligibility_reason=None,
+ billing_tier=BillingTier.SELF_SERVE,
+ first_version_created_at=None,
+ )
+
+ assert scoped_context.workflow_block_engine_override is None
+ assert resolution.log["route_reason"] == WorkflowBlockEngineRouteReason.flag_bucket_control