Skip to content

Automate browser based workflows with AI

Point the Terms model-training opt-out to an enterprise agreement and [email protected] (SKY-17923) (#8862)

Shuchang Zheng5cd25fc6f0ab
diff --git a/skyvern/config.py b/skyvern/config.py
index 6b230ed06..f581af694 100644
--- a/skyvern/config.py
+++ b/skyvern/config.py
@@ -583,7 +583,8 @@ class Settings(BaseSettings):
# are not randomized, so every per-arm read must exclude them by route_reason.
TASK_V3_DEFAULT_ENGINE_WORKFLOW_CUTOFF: datetime | None = None
# Workflows whose permanent id was born at or after this instant, in every billing tier, honor each
- # task block's chosen engine and run a block with no engine on Task V3, outside WORKFLOW_TASK_V3_AB.
+ # task block's chosen engine and run a block with no engine on Task V3 (or on the per-run split of
+ # WORKFLOW_TASK_V3_AB_NEW_WORKFLOWS for a listed org or workflow), outside WORKFLOW_TASK_V3_AB.
# Excludes the same per-call platform workflows as the cutoff above. None disables the rule, as does
# having no flag provider; DISABLE_TASK_V3 still wins. A naive value is read as UTC.
TASK_V3_CHOSEN_ENGINE_CUTOFF: datetime | None = None
diff --git a/skyvern/forge/sdk/experimentation/workflow_block_engine.py b/skyvern/forge/sdk/experimentation/workflow_block_engine.py
index a0b1ee059..018620154 100644
--- a/skyvern/forge/sdk/experimentation/workflow_block_engine.py
+++ b/skyvern/forge/sdk/experimentation/workflow_block_engine.py
@@ -23,6 +23,7 @@ if TYPE_CHECKING:
LOG = structlog.get_logger()
WORKFLOW_TASK_V3_AB_FLAG = "WORKFLOW_TASK_V3_AB"
+WORKFLOW_TASK_V3_AB_NEW_WORKFLOWS_FLAG = "WORKFLOW_TASK_V3_AB_NEW_WORKFLOWS"
WOULD_BE_CODE_PROPERTY = "would_be_code"
DISABLE_TASK_V3_FLAG = "DISABLE_TASK_V3"
@@ -55,6 +56,10 @@ class WorkflowBlockEngineRouteReason(StrEnum):
# left its engine unset and runs on v3 (``new_workflow_v3_default``), or every block's engine was
# chosen and is honored as written (``chosen_engine``).
new_workflow_v3_default = "new_workflow_v3_default"
+ # The same workflows when WORKFLOW_TASK_V3_AB_NEW_WORKFLOWS lists the run's org or workflow: its Default blocks
+ # are randomized per run, v3 in treatment and v1 in control.
+ new_workflow_ab_treatment = "new_workflow_ab_treatment"
+ new_workflow_ab_control = "new_workflow_ab_control"
chosen_engine = "chosen_engine"
# Before that cutoff, a block a person pinned to skyvern-1.0 keeps the run out of the A/B; labelled
# apart from ``ineligible`` so reads can count and drop these runs by this value alone.
@@ -237,6 +242,35 @@ async def _ab_flag_puts_run_in_treatment(
)
+async def _new_workflow_ab_variant(
+ provider: BaseExperimentationProvider,
+ *,
+ workflow_run_id: str,
+ organization_id: str | None,
+ workflow_permanent_id: str | None,
+) -> str | None:
+ """The run's WORKFLOW_TASK_V3_AB_NEW_WORKFLOWS variant, or None for a missing flag, an unlisted run or a raised
+ read. Only "v1" and "v3" move the run; the caller keeps v3 for anything else.
+ """
+ try:
+ variant = await provider.get_value_cached(
+ WORKFLOW_TASK_V3_AB_NEW_WORKFLOWS_FLAG,
+ workflow_run_id,
+ properties={
+ "organization_id": organization_id,
+ "workflow_permanent_id": workflow_permanent_id or "not_workflow",
+ },
+ )
+ except Exception:
+ LOG.warning(
+ "Failed to read WORKFLOW_TASK_V3_AB_NEW_WORKFLOWS; keeping v3",
+ workflow_run_id=workflow_run_id,
+ exc_info=True,
+ )
+ return None
+ return variant
+
+
async def resolve_workflow_block_engine_arm(
context: skyvern_context.SkyvernContext,
*,
@@ -270,6 +304,8 @@ async def resolve_workflow_block_engine_arm(
engine unset, or is None when the run has no engine to choose. The rule does not require A/B
eligibility, since a pinned block no longer keeps the run's unset blocks on v1, and so covers a
script run's uncached blocks and AI fallback.
+ ``WORKFLOW_TASK_V3_AB_NEW_WORKFLOWS`` can randomize those unset blocks per run for a listed org or
+ workflow: variant ``v1`` leaves the override unset, ``v3`` sets it, and anything else keeps v3.
``workflow_status`` is the status of the version this run executes and ``trigger_type`` is how
the run was launched. Together they are what separates a workflow a customer keeps from the
@@ -337,8 +373,22 @@ async def resolve_workflow_block_engine_arm(
elif chosen_engine_candidate and await born_at_or_after(settings.TASK_V3_CHOSEN_ENGINE_CUTOFF):
honors_chosen_engine = True
if takes_default_engine:
- override = RunEngine.skyvern_v3
- route_reason = WorkflowBlockEngineRouteReason.new_workflow_v3_default
+ variant = await _new_workflow_ab_variant(
+ provider,
+ workflow_run_id=workflow_run_id,
+ organization_id=organization_id,
+ workflow_permanent_id=workflow_permanent_id,
+ )
+ if variant == "v1":
+ # Control leaves the override unset, so Default blocks run v1 and pins keep their engine.
+ route_reason = WorkflowBlockEngineRouteReason.new_workflow_ab_control
+ else:
+ override = RunEngine.skyvern_v3
+ route_reason = (
+ WorkflowBlockEngineRouteReason.new_workflow_ab_treatment
+ if variant == "v3"
+ else WorkflowBlockEngineRouteReason.new_workflow_v3_default
+ )
else:
route_reason = WorkflowBlockEngineRouteReason.chosen_engine
elif run_is_eligible:
diff --git a/tests/unit/_workflow_block_engine_fakes.py b/tests/unit/_workflow_block_engine_fakes.py
index 28e331b56..00c46fbb1 100644
--- a/tests/unit/_workflow_block_engine_fakes.py
+++ b/tests/unit/_workflow_block_engine_fakes.py
@@ -39,6 +39,7 @@ class FakeExperimentationProvider(BaseExperimentationProvider):
raise_error: bool = False,
strict_error_flags: set[str] | None = None,
unresolvable_flags: set[str] | None = None,
+ values: dict[str, str] | None = None,
) -> None:
super().__init__()
self.flags = dict(flags or {})
@@ -49,6 +50,8 @@ class FakeExperimentationProvider(BaseExperimentationProvider):
# carrying a condition only the PostHog API can resolve. Neither resolver raises; both
# return None.
self.unresolvable_flags = set(unresolvable_flags or ())
+ # Multivariate answers, read through get_value; a flag in strict_error_flags raises there too.
+ self.values = dict(values or {})
async def _prepare_feature_flag_resolution(self, feature_name: str, *, cached: bool) -> None:
# Where a provider does raise in production: the local provider reloads its flag snapshot
@@ -81,7 +84,10 @@ class FakeExperimentationProvider(BaseExperimentationProvider):
return bool(await self._resolve_feature_flag(feature_name, distinct_id, properties))
async def _get_value(self, feature_name: str, distinct_id: str, properties: dict | None = None) -> str | None:
- return None
+ self.calls.append((feature_name, distinct_id, properties))
+ if feature_name in self.strict_error_flags:
+ raise RuntimeError("provider unavailable")
+ return self.values.get(feature_name)
async def _get_payload(self, feature_name: str, distinct_id: str, properties: dict | None = None) -> Any:
return None
diff --git a/tests/unit/test_workflow_block_engine.py b/tests/unit/test_workflow_block_engine.py
index adfe4fd69..0e28059b8 100644
--- a/tests/unit/test_workflow_block_engine.py
+++ b/tests/unit/test_workflow_block_engine.py
@@ -25,6 +25,7 @@ from skyvern.forge.sdk.experimentation.providers import BaseExperimentationProvi
from skyvern.forge.sdk.experimentation.workflow_block_engine import (
DISABLE_TASK_V3_FLAG,
WORKFLOW_TASK_V3_AB_FLAG,
+ WORKFLOW_TASK_V3_AB_NEW_WORKFLOWS_FLAG,
WOULD_BE_CODE_PROPERTY,
WorkflowBlockEngineRouteReason,
_as_utc,
@@ -1479,11 +1480,17 @@ async def _resolve_chosen_engine_run(
born_at: datetime,
flags: dict[str, bool] | None = None,
workflow_status: WorkflowStatus = WorkflowStatus.published,
+ values: dict[str, str] | None = None,
+ strict_error_flags: set[str] | None = None,
) -> tuple[FakeExperimentationProvider, Any]:
# Eligibility and the unset-engine fact are computed from the blocks by the same helpers the
# workflow service calls, so a block-level regression in either reaches these assertions.
monkeypatch.setattr(settings, "TASK_V3_CHOSEN_ENGINE_CUTOFF", cutoff)
- provider = FakeExperimentationProvider(flags if flags is not None else {WORKFLOW_TASK_V3_AB_FLAG: True})
+ provider = FakeExperimentationProvider(
+ flags if flags is not None else {WORKFLOW_TASK_V3_AB_FLAG: True},
+ values=values,
+ strict_error_flags=strict_error_flags,
+ )
resolution = await resolve_arm(
context,
provider,
@@ -1540,8 +1547,9 @@ async def test_chosen_engine_cutoff_matrix(
assert block.resolve_engine("wr_matrix") == expected_engine
assert resolution.log["route_reason"] == expected_reason
if expected_reason in (_R.new_workflow_v3_default, _R.chosen_engine):
- # Every tier, with no A/B draw and no billing-tier lookup.
- assert consulted_flags(provider) == [DISABLE_TASK_V3_FLAG]
+ # Every tier, with no A/B draw and no billing-tier lookup; only an unset block asks the new-workflow split.
+ new_split = [WORKFLOW_TASK_V3_AB_NEW_WORKFLOWS_FLAG] if expected_reason == _R.new_workflow_v3_default else []
+ assert consulted_flags(provider) == [DISABLE_TASK_V3_FLAG, *new_split]
assert resolution.log["billing_tier"] is None
@@ -1569,6 +1577,87 @@ async def test_a_post_cutoff_run_honors_each_block_on_its_own(
assert resolution.log["route_reason"] == _R.new_workflow_v3_default
+ ("values", "strict_error_flags", "expected_engine", "expected_reason"),
+ [
+ ({}, None, RunEngine.skyvern_v3, _R.new_workflow_v3_default),
+ ({"WORKFLOW_TASK_V3_AB_NEW_WORKFLOWS": "v3"}, None, RunEngine.skyvern_v3, _R.new_workflow_ab_treatment),
+ ({"WORKFLOW_TASK_V3_AB_NEW_WORKFLOWS": "v1"}, None, RunEngine.skyvern_v1, _R.new_workflow_ab_control),
+ ({"WORKFLOW_TASK_V3_AB_NEW_WORKFLOWS": "50"}, None, RunEngine.skyvern_v3, _R.new_workflow_v3_default),
+ ({}, {"WORKFLOW_TASK_V3_AB_NEW_WORKFLOWS"}, RunEngine.skyvern_v3, _R.new_workflow_v3_default),
+ ],
+ ids=["not-listed", "treatment", "control", "unknown-variant", "raised-read"],
+)
+async def test_the_new_workflow_split_moves_only_unset_blocks(
+ scoped_context: SkyvernContext,
+ monkeypatch: pytest.MonkeyPatch,
+ values: dict[str, str],
+ strict_error_flags: set[str] | None,
+ expected_engine: RunEngine,
+ expected_reason: WorkflowBlockEngineRouteReason,
+) -> None:
+ # Only a listed run's control variant moves its unset blocks to v1; a v3 pin stays v3 in every cell, and anything
+ # that is not a listed variant keeps today's v3 default.
+ unset = _make_block(NavigationBlock, label="unset", navigation_goal="Apply to the job")
+ chose_v3 = _make_block(TaskBlock, label="chose_v3", engine=RunEngine.skyvern_v3)
+ provider, resolution = await _resolve_chosen_engine_run(
+ scoped_context,
+ monkeypatch,
+ [unset, chose_v3],
+ run_id="wr_new_split",
+ cutoff=CHOSEN_ENGINE_CUTOFF,
+ born_at=_AFTER,
+ values=values,
+ strict_error_flags=strict_error_flags,
+ )
+
+ assert [unset.resolve_engine("wr_new_split"), chose_v3.resolve_engine("wr_new_split")] == [
+ expected_engine,
+ RunEngine.skyvern_v3,
+ ]
+ assert resolution.log["route_reason"] == expected_reason
+ assert WORKFLOW_TASK_V3_AB_FLAG not in consulted_flags(provider)
+ (call,) = (c for c in provider.calls if c[0] == WORKFLOW_TASK_V3_AB_NEW_WORKFLOWS_FLAG)
+ # Both properties are what a release condition can list; a segment listed by organization never matches without it.
+ assert call[1:] == ("wr_new_split", {"organization_id": "org_1", "workflow_permanent_id": "wpid_1"})
+
+
+async def test_the_new_workflow_split_never_reaches_a_pre_cutoff_or_killed_run(
+ scoped_context: SkyvernContext, monkeypatch: pytest.MonkeyPatch
+) -> None:
+ control = {"WORKFLOW_TASK_V3_AB_NEW_WORKFLOWS": "v1"}
+ before = _make_block(TaskBlock, label="before")
+ provider, resolution = await _resolve_chosen_engine_run(
+ scoped_context,
+ monkeypatch,
+ [before],
+ run_id="wr_pre",
+ cutoff=CHOSEN_ENGINE_CUTOFF,
+ born_at=_BEFORE,
+ values=control,
+ )
+ assert before.resolve_engine("wr_pre") == RunEngine.skyvern_v3
+ assert resolution.log["route_reason"] == _R.flag_bucket_treatment
+ assert WORKFLOW_TASK_V3_AB_NEW_WORKFLOWS_FLAG not in consulted_flags(provider)
+
+ killed = _make_block(TaskBlock, label="killed")
+ provider, resolution = await _resolve_chosen_engine_run(
+ scoped_context,
+ monkeypatch,
+ [killed],
+ run_id="wr_killed_split",
+ cutoff=CHOSEN_ENGINE_CUTOFF,
+ born_at=_AFTER,
+ flags={WORKFLOW_TASK_V3_AB_FLAG: True, DISABLE_TASK_V3_FLAG: True},
+ values={"WORKFLOW_TASK_V3_AB_NEW_WORKFLOWS": "v3"},
+ )
+ assert killed.resolve_engine("wr_killed_split") == RunEngine.skyvern_v1
+ assert resolution.log["route_reason"] == _R.disabled
+ assert WORKFLOW_TASK_V3_AB_NEW_WORKFLOWS_FLAG not in consulted_flags(provider)
+
+
@pytest.mark.parametrize("chosen", [None, RunEngine.skyvern_v3])
@pytest.mark.asyncio
async def test_kill_switch_wins_over_the_chosen_engine_cutoff(
@@ -1829,6 +1918,27 @@ async def test_the_new_workflow_defaults_never_take_a_code_mode_run_off_its_scri
assert code_mode_displaced("wr_chosen_code") is False
+async def test_a_new_workflow_code_mode_run_on_control_falls_back_to_v1_and_keeps_its_script(
+ scoped_context: SkyvernContext, monkeypatch: pytest.MonkeyPatch
+) -> None:
+ # A script run's uncached blocks and AI fallback resolve through the block, so they follow the split's arm.
+ monkeypatch.setattr(settings, "TASK_V3_CHOSEN_ENGINE_CUTOFF", CHOSEN_ENGINE_CUTOFF)
+ resolution = await resolve_arm(
+ scoped_context,
+ FakeExperimentationProvider(
+ {WORKFLOW_TASK_V3_AB_FLAG: True}, values={WORKFLOW_TASK_V3_AB_NEW_WORKFLOWS_FLAG: "v1"}
+ ),
+ workflow_run_id="wr_new_code_control",
+ ineligibility_reason=None,
+ first_version_created_at=_AFTER,
+ would_be_code=True,
+ )
+ assert resolution.log["route_reason"] == _R.new_workflow_ab_control
+ assert _make_block(TaskBlock, label="uncached").resolve_engine("wr_new_code_control") == RunEngine.skyvern_v1
+ assert code_mode_displaced("wr_new_code_control") is False
+
+
@pytest.mark.asyncio
async def test_a_treatment_code_mode_run_executes_on_v3_without_loading_or_minting_a_script(
scoped_context: SkyvernContext, monkeypatch: pytest.MonkeyPatch