Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
Point the Terms model-training opt-out to an enterprise agreement and [email protected] (SKY-17923) (#8862)
Shuchang Zheng5cd25fc6f0ab
diff --git a/skyvern/config.py b/skyvern/config.pyindex 6b230ed06..f581af694 100644--- a/skyvern/config.py+++ b/skyvern/config.py@@ -583,7 +583,8 @@ class Settings(BaseSettings):# are not randomized, so every per-arm read must exclude them by route_reason.TASK_V3_DEFAULT_ENGINE_WORKFLOW_CUTOFF: datetime | None = None# Workflows whose permanent id was born at or after this instant, in every billing tier, honor each- # task block's chosen engine and run a block with no engine on Task V3, outside WORKFLOW_TASK_V3_AB.+ # task block's chosen engine and run a block with no engine on Task V3 (or on the per-run split of+ # WORKFLOW_TASK_V3_AB_NEW_WORKFLOWS for a listed org or workflow), outside WORKFLOW_TASK_V3_AB.# Excludes the same per-call platform workflows as the cutoff above. None disables the rule, as does# having no flag provider; DISABLE_TASK_V3 still wins. A naive value is read as UTC.TASK_V3_CHOSEN_ENGINE_CUTOFF: datetime | None = Nonediff --git a/skyvern/forge/sdk/experimentation/workflow_block_engine.py b/skyvern/forge/sdk/experimentation/workflow_block_engine.pyindex a0b1ee059..018620154 100644--- a/skyvern/forge/sdk/experimentation/workflow_block_engine.py+++ b/skyvern/forge/sdk/experimentation/workflow_block_engine.py@@ -23,6 +23,7 @@ if TYPE_CHECKING:LOG = structlog.get_logger()WORKFLOW_TASK_V3_AB_FLAG = "WORKFLOW_TASK_V3_AB"+WORKFLOW_TASK_V3_AB_NEW_WORKFLOWS_FLAG = "WORKFLOW_TASK_V3_AB_NEW_WORKFLOWS"WOULD_BE_CODE_PROPERTY = "would_be_code"DISABLE_TASK_V3_FLAG = "DISABLE_TASK_V3"@@ -55,6 +56,10 @@ class WorkflowBlockEngineRouteReason(StrEnum):# left its engine unset and runs on v3 (``new_workflow_v3_default``), or every block's engine was# chosen and is honored as written (``chosen_engine``).new_workflow_v3_default = "new_workflow_v3_default"+ # The same workflows when WORKFLOW_TASK_V3_AB_NEW_WORKFLOWS lists the run's org or workflow: its Default blocks+ # are randomized per run, v3 in treatment and v1 in control.+ new_workflow_ab_treatment = "new_workflow_ab_treatment"+ new_workflow_ab_control = "new_workflow_ab_control"chosen_engine = "chosen_engine"# Before that cutoff, a block a person pinned to skyvern-1.0 keeps the run out of the A/B; labelled# apart from ``ineligible`` so reads can count and drop these runs by this value alone.@@ -237,6 +242,35 @@ async def _ab_flag_puts_run_in_treatment()+async def _new_workflow_ab_variant(+ provider: BaseExperimentationProvider,+ *,+ workflow_run_id: str,+ organization_id: str | None,+ workflow_permanent_id: str | None,+) -> str | None:+ """The run's WORKFLOW_TASK_V3_AB_NEW_WORKFLOWS variant, or None for a missing flag, an unlisted run or a raised+ read. Only "v1" and "v3" move the run; the caller keeps v3 for anything else.+ """+ try:+ variant = await provider.get_value_cached(+ WORKFLOW_TASK_V3_AB_NEW_WORKFLOWS_FLAG,+ workflow_run_id,+ properties={+ "organization_id": organization_id,+ "workflow_permanent_id": workflow_permanent_id or "not_workflow",+ },+ )+ except Exception:+ LOG.warning(+ "Failed to read WORKFLOW_TASK_V3_AB_NEW_WORKFLOWS; keeping v3",+ workflow_run_id=workflow_run_id,+ exc_info=True,+ )+ return None+ return variant++async def resolve_workflow_block_engine_arm(context: skyvern_context.SkyvernContext,*,@@ -270,6 +304,8 @@ async def resolve_workflow_block_engine_arm(engine unset, or is None when the run has no engine to choose. The rule does not require A/Beligibility, since a pinned block no longer keeps the run's unset blocks on v1, and so covers ascript run's uncached blocks and AI fallback.+ ``WORKFLOW_TASK_V3_AB_NEW_WORKFLOWS`` can randomize those unset blocks per run for a listed org or+ workflow: variant ``v1`` leaves the override unset, ``v3`` sets it, and anything else keeps v3.``workflow_status`` is the status of the version this run executes and ``trigger_type`` is howthe run was launched. Together they are what separates a workflow a customer keeps from the@@ -337,8 +373,22 @@ async def resolve_workflow_block_engine_arm(elif chosen_engine_candidate and await born_at_or_after(settings.TASK_V3_CHOSEN_ENGINE_CUTOFF):honors_chosen_engine = Trueif takes_default_engine:- override = RunEngine.skyvern_v3- route_reason = WorkflowBlockEngineRouteReason.new_workflow_v3_default+ variant = await _new_workflow_ab_variant(+ provider,+ workflow_run_id=workflow_run_id,+ organization_id=organization_id,+ workflow_permanent_id=workflow_permanent_id,+ )+ if variant == "v1":+ # Control leaves the override unset, so Default blocks run v1 and pins keep their engine.+ route_reason = WorkflowBlockEngineRouteReason.new_workflow_ab_control+ else:+ override = RunEngine.skyvern_v3+ route_reason = (+ WorkflowBlockEngineRouteReason.new_workflow_ab_treatment+ if variant == "v3"+ else WorkflowBlockEngineRouteReason.new_workflow_v3_default+ )else:route_reason = WorkflowBlockEngineRouteReason.chosen_engineelif run_is_eligible:diff --git a/tests/unit/_workflow_block_engine_fakes.py b/tests/unit/_workflow_block_engine_fakes.pyindex 28e331b56..00c46fbb1 100644--- a/tests/unit/_workflow_block_engine_fakes.py+++ b/tests/unit/_workflow_block_engine_fakes.py@@ -39,6 +39,7 @@ class FakeExperimentationProvider(BaseExperimentationProvider):raise_error: bool = False,strict_error_flags: set[str] | None = None,unresolvable_flags: set[str] | None = None,+ values: dict[str, str] | None = None,) -> None:super().__init__()self.flags = dict(flags or {})@@ -49,6 +50,8 @@ class FakeExperimentationProvider(BaseExperimentationProvider):# carrying a condition only the PostHog API can resolve. Neither resolver raises; both# return None.self.unresolvable_flags = set(unresolvable_flags or ())+ # Multivariate answers, read through get_value; a flag in strict_error_flags raises there too.+ self.values = dict(values or {})async def _prepare_feature_flag_resolution(self, feature_name: str, *, cached: bool) -> None:# Where a provider does raise in production: the local provider reloads its flag snapshot@@ -81,7 +84,10 @@ class FakeExperimentationProvider(BaseExperimentationProvider):return bool(await self._resolve_feature_flag(feature_name, distinct_id, properties))async def _get_value(self, feature_name: str, distinct_id: str, properties: dict | None = None) -> str | None:- return None+ self.calls.append((feature_name, distinct_id, properties))+ if feature_name in self.strict_error_flags:+ raise RuntimeError("provider unavailable")+ return self.values.get(feature_name)async def _get_payload(self, feature_name: str, distinct_id: str, properties: dict | None = None) -> Any:return Nonediff --git a/tests/unit/test_workflow_block_engine.py b/tests/unit/test_workflow_block_engine.pyindex adfe4fd69..0e28059b8 100644--- a/tests/unit/test_workflow_block_engine.py+++ b/tests/unit/test_workflow_block_engine.py@@ -25,6 +25,7 @@ from skyvern.forge.sdk.experimentation.providers import BaseExperimentationProvifrom skyvern.forge.sdk.experimentation.workflow_block_engine import (DISABLE_TASK_V3_FLAG,WORKFLOW_TASK_V3_AB_FLAG,+ WORKFLOW_TASK_V3_AB_NEW_WORKFLOWS_FLAG,WOULD_BE_CODE_PROPERTY,WorkflowBlockEngineRouteReason,_as_utc,@@ -1479,11 +1480,17 @@ async def _resolve_chosen_engine_run(born_at: datetime,flags: dict[str, bool] | None = None,workflow_status: WorkflowStatus = WorkflowStatus.published,+ values: dict[str, str] | None = None,+ strict_error_flags: set[str] | None = None,) -> tuple[FakeExperimentationProvider, Any]:# Eligibility and the unset-engine fact are computed from the blocks by the same helpers the# workflow service calls, so a block-level regression in either reaches these assertions.monkeypatch.setattr(settings, "TASK_V3_CHOSEN_ENGINE_CUTOFF", cutoff)- provider = FakeExperimentationProvider(flags if flags is not None else {WORKFLOW_TASK_V3_AB_FLAG: True})+ provider = FakeExperimentationProvider(+ flags if flags is not None else {WORKFLOW_TASK_V3_AB_FLAG: True},+ values=values,+ strict_error_flags=strict_error_flags,+ )resolution = await resolve_arm(context,provider,@@ -1540,8 +1547,9 @@ async def test_chosen_engine_cutoff_matrix(assert block.resolve_engine("wr_matrix") == expected_engineassert resolution.log["route_reason"] == expected_reasonif expected_reason in (_R.new_workflow_v3_default, _R.chosen_engine):- # Every tier, with no A/B draw and no billing-tier lookup.- assert consulted_flags(provider) == [DISABLE_TASK_V3_FLAG]+ # Every tier, with no A/B draw and no billing-tier lookup; only an unset block asks the new-workflow split.+ new_split = [WORKFLOW_TASK_V3_AB_NEW_WORKFLOWS_FLAG] if expected_reason == _R.new_workflow_v3_default else []+ assert consulted_flags(provider) == [DISABLE_TASK_V3_FLAG, *new_split]assert resolution.log["billing_tier"] is None@@ -1569,6 +1577,87 @@ async def test_a_post_cutoff_run_honors_each_block_on_its_own(assert resolution.log["route_reason"] == _R.new_workflow_v3_default+ ("values", "strict_error_flags", "expected_engine", "expected_reason"),+ [+ ({}, None, RunEngine.skyvern_v3, _R.new_workflow_v3_default),+ ({"WORKFLOW_TASK_V3_AB_NEW_WORKFLOWS": "v3"}, None, RunEngine.skyvern_v3, _R.new_workflow_ab_treatment),+ ({"WORKFLOW_TASK_V3_AB_NEW_WORKFLOWS": "v1"}, None, RunEngine.skyvern_v1, _R.new_workflow_ab_control),+ ({"WORKFLOW_TASK_V3_AB_NEW_WORKFLOWS": "50"}, None, RunEngine.skyvern_v3, _R.new_workflow_v3_default),+ ({}, {"WORKFLOW_TASK_V3_AB_NEW_WORKFLOWS"}, RunEngine.skyvern_v3, _R.new_workflow_v3_default),+ ],+ ids=["not-listed", "treatment", "control", "unknown-variant", "raised-read"],+)+async def test_the_new_workflow_split_moves_only_unset_blocks(+ scoped_context: SkyvernContext,+ monkeypatch: pytest.MonkeyPatch,+ values: dict[str, str],+ strict_error_flags: set[str] | None,+ expected_engine: RunEngine,+ expected_reason: WorkflowBlockEngineRouteReason,+) -> None:+ # Only a listed run's control variant moves its unset blocks to v1; a v3 pin stays v3 in every cell, and anything+ # that is not a listed variant keeps today's v3 default.+ unset = _make_block(NavigationBlock, label="unset", navigation_goal="Apply to the job")+ chose_v3 = _make_block(TaskBlock, label="chose_v3", engine=RunEngine.skyvern_v3)+ provider, resolution = await _resolve_chosen_engine_run(+ scoped_context,+ monkeypatch,+ [unset, chose_v3],+ run_id="wr_new_split",+ cutoff=CHOSEN_ENGINE_CUTOFF,+ born_at=_AFTER,+ values=values,+ strict_error_flags=strict_error_flags,+ )++ assert [unset.resolve_engine("wr_new_split"), chose_v3.resolve_engine("wr_new_split")] == [+ expected_engine,+ RunEngine.skyvern_v3,+ ]+ assert resolution.log["route_reason"] == expected_reason+ assert WORKFLOW_TASK_V3_AB_FLAG not in consulted_flags(provider)+ (call,) = (c for c in provider.calls if c[0] == WORKFLOW_TASK_V3_AB_NEW_WORKFLOWS_FLAG)+ # Both properties are what a release condition can list; a segment listed by organization never matches without it.+ assert call[1:] == ("wr_new_split", {"organization_id": "org_1", "workflow_permanent_id": "wpid_1"})+++async def test_the_new_workflow_split_never_reaches_a_pre_cutoff_or_killed_run(+ scoped_context: SkyvernContext, monkeypatch: pytest.MonkeyPatch+) -> None:+ control = {"WORKFLOW_TASK_V3_AB_NEW_WORKFLOWS": "v1"}+ before = _make_block(TaskBlock, label="before")+ provider, resolution = await _resolve_chosen_engine_run(+ scoped_context,+ monkeypatch,+ [before],+ run_id="wr_pre",+ cutoff=CHOSEN_ENGINE_CUTOFF,+ born_at=_BEFORE,+ values=control,+ )+ assert before.resolve_engine("wr_pre") == RunEngine.skyvern_v3+ assert resolution.log["route_reason"] == _R.flag_bucket_treatment+ assert WORKFLOW_TASK_V3_AB_NEW_WORKFLOWS_FLAG not in consulted_flags(provider)++ killed = _make_block(TaskBlock, label="killed")+ provider, resolution = await _resolve_chosen_engine_run(+ scoped_context,+ monkeypatch,+ [killed],+ run_id="wr_killed_split",+ cutoff=CHOSEN_ENGINE_CUTOFF,+ born_at=_AFTER,+ flags={WORKFLOW_TASK_V3_AB_FLAG: True, DISABLE_TASK_V3_FLAG: True},+ values={"WORKFLOW_TASK_V3_AB_NEW_WORKFLOWS": "v3"},+ )+ assert killed.resolve_engine("wr_killed_split") == RunEngine.skyvern_v1+ assert resolution.log["route_reason"] == _R.disabled+ assert WORKFLOW_TASK_V3_AB_NEW_WORKFLOWS_FLAG not in consulted_flags(provider)++@pytest.mark.parametrize("chosen", [None, RunEngine.skyvern_v3])@pytest.mark.asyncioasync def test_kill_switch_wins_over_the_chosen_engine_cutoff(@@ -1829,6 +1918,27 @@ async def test_the_new_workflow_defaults_never_take_a_code_mode_run_off_its_scriassert code_mode_displaced("wr_chosen_code") is False+async def test_a_new_workflow_code_mode_run_on_control_falls_back_to_v1_and_keeps_its_script(+ scoped_context: SkyvernContext, monkeypatch: pytest.MonkeyPatch+) -> None:+ # A script run's uncached blocks and AI fallback resolve through the block, so they follow the split's arm.+ monkeypatch.setattr(settings, "TASK_V3_CHOSEN_ENGINE_CUTOFF", CHOSEN_ENGINE_CUTOFF)+ resolution = await resolve_arm(+ scoped_context,+ FakeExperimentationProvider(+ {WORKFLOW_TASK_V3_AB_FLAG: True}, values={WORKFLOW_TASK_V3_AB_NEW_WORKFLOWS_FLAG: "v1"}+ ),+ workflow_run_id="wr_new_code_control",+ ineligibility_reason=None,+ first_version_created_at=_AFTER,+ would_be_code=True,+ )+ assert resolution.log["route_reason"] == _R.new_workflow_ab_control+ assert _make_block(TaskBlock, label="uncached").resolve_engine("wr_new_code_control") == RunEngine.skyvern_v1+ assert code_mode_displaced("wr_new_code_control") is False++@pytest.mark.asyncioasync def test_a_treatment_code_mode_run_executes_on_v3_without_loading_or_minting_a_script(scoped_context: SkyvernContext, monkeypatch: pytest.MonkeyPatch