Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
Replace the Workday static script (deterministic) + flag-gated code-mode rollout (#7603)
Cindy Li62d14e9fbf86
diff --git a/skyvern/forge/agent_functions.py b/skyvern/forge/agent_functions.pyindex 17fa33e35..7297082c3 100644--- a/skyvern/forge/agent_functions.py+++ b/skyvern/forge/agent_functions.py@@ -802,6 +802,14 @@ class AgentFunction:) -> bool:return True+ async def should_upgrade_to_code_mode(+ self,+ *,+ workflow: "Workflow",+ workflow_run: "WorkflowRun",+ ) -> bool:+ return False+async def resolve_mcp_code_only_mode(self,organization_id: str | None,diff --git a/skyvern/forge/sdk/workflow/code_mode_fallback.py b/skyvern/forge/sdk/workflow/code_mode_fallback.pynew file mode 100644index 000000000..99272c8ac--- /dev/null+++ b/skyvern/forge/sdk/workflow/code_mode_fallback.py@@ -0,0 +1,217 @@+from __future__ import annotations++import structlog+from sqlalchemy.exc import IntegrityError++from skyvern.exceptions import WorkflowNotFound+from skyvern.forge import app+from skyvern.forge.sdk.core import skyvern_context+from skyvern.forge.sdk.core.skyvern_context import SkyvernContext+from skyvern.forge.sdk.workflow.models.validators import is_reserved_tag_key+from skyvern.forge.sdk.workflow.models.workflow import WorkflowRun, WorkflowRunStatus++LOG = structlog.get_logger()++CODE_MODE_FALLBACK_RETRY_FLAG = "CODE_MODE_FALLBACK_RETRY"+CODE_RUN_WITH = "code"+AGENT_RUN_WITH = "agent"+# A code-mode run that failed/terminated retries once as a pure-agent run. timed_out is excluded:+# re-running the full (slower) agent path on an already-long run is rarely what the caller wants.+#+# Double-apply risk (accepted, flag-gated): the retry is a fresh full-workflow run, so a run that+# submitted an application and then failed on a later block would submit again. A run that actually+# completed never reaches here (only failed/terminated retry), which covers the common case; the+# submitted-then-errored edge is real, so the feature stays off by default (CODE_MODE_FALLBACK_RETRY)+# and is enabled per-org only with explicit owner sign-off that the target tolerates a possible+# duplicate submission.+RETRYABLE_STATUSES = frozenset({WorkflowRunStatus.failed, WorkflowRunStatus.terminated})+++async def _retry_enabled_for_organization(organization_id: str, workflow_run: WorkflowRun) -> bool:+ """Gate the auto-retry rollout. Fails closed: a retry run costs credits and re-fires webhooks.++ distinct_id = organization_id (deliberately NOT run-level), so an org is fully in-or-out+ rather than retrying on a random subset of its runs. Mirrors credential_fallback.+ """+ provider = getattr(app, "EXPERIMENTATION_PROVIDER", None)+ if not provider:+ return False+ try:+ return bool(+ await provider.is_feature_enabled_cached(+ CODE_MODE_FALLBACK_RETRY_FLAG,+ organization_id,+ properties={+ "organization_id": organization_id,+ "workflow_permanent_id": workflow_run.workflow_permanent_id,+ },+ )+ )+ except Exception:+ LOG.warning(+ "code_mode_fallback.flag_error",+ workflow_run_id=workflow_run.workflow_run_id,+ organization_id=organization_id,+ exc_info=True,+ )+ return False+++def _trigger_matches(workflow_run: WorkflowRun) -> bool:+ """A code-mode run in a retryable terminal state is eligible; anything else is not."""+ if workflow_run.status not in RETRYABLE_STATUSES:+ return False+ return (workflow_run.run_with or AGENT_RUN_WITH) == CODE_RUN_WITH+++async def _reload_user_run_metadata(workflow_run: WorkflowRun, organization_id: str) -> dict[str, str] | None:+ """Reload the failed run's user-writable tags so the retry keeps them. Reserved keys are dropped —+ they are re-derived per run. Mirrors credential_fallback / the manual retry path."""+ try:+ grouped = await app.DATABASE.tags.get_active_grouped_tags_for_run(+ workflow_run_id=workflow_run.workflow_run_id,+ organization_id=organization_id,+ )+ except Exception:+ LOG.warning(+ "Failed to reload run metadata for code-mode fallback retry; continuing without it",+ workflow_run_id=workflow_run.workflow_run_id,+ exc_info=True,+ )+ return None+ if not grouped:+ return None+ return {key: value for key, value in grouped.items() if not is_reserved_tag_key(key)} or None+++async def maybe_start_code_mode_fallback_retry(workflow_run: WorkflowRun, organization_id: str) -> str | None:+ try:+ return await _maybe_start_code_mode_fallback_retry(workflow_run, organization_id)+ except Exception:+ LOG.warning(+ "Failed to start code-mode fallback retry",+ workflow_run_id=workflow_run.workflow_run_id,+ organization_id=organization_id,+ exc_info=True,+ )+ return None+++async def _maybe_start_code_mode_fallback_retry(workflow_run: WorkflowRun, organization_id: str) -> str | None:+ # Scheduled from clean_up_workflow, which runs for every terminal status. Bail on ineligible runs+ # before the flag check and DB reads so the success path stays cheap.+ if not _trigger_matches(workflow_run):+ return None++ # Never retry a retry: an agent attempt that fails is terminal, and a run that is already a+ # fallback retry (credential or code) must not spawn another. This also bounds the chain.+ if workflow_run.retried_from_workflow_run_id or workflow_run.fallback_attempt:+ return None++ # Block-run / copilot-test / debug runs supply block_labels directly and must not spawn a+ # detached full-workflow run (real credits, real side effects) outside the test.+ if workflow_run.parent_workflow_run_id or workflow_run.debug_session_id or workflow_run.copilot_session_id:+ return None++ if not await _retry_enabled_for_organization(organization_id, workflow_run):+ return None++ is_block_scoped_run = await app.AGENT_FUNCTION.is_block_scoped_workflow_run(workflow_run)+ if not is_block_scoped_run:+ is_block_scoped_run = await app.DATABASE.debug.has_block_run_for_workflow_run(+ organization_id=organization_id,+ workflow_run_id=workflow_run.workflow_run_id,+ )+ if is_block_scoped_run:+ return None++ # Idempotency: the retried_from_workflow_run_id unique index allows one retry per run, shared+ # with credential_fallback — so if credential fallback already claimed the slot, no-op.+ existing_retry_run_id = await app.DATABASE.workflow_runs.get_workflow_run_retried_by(+ workflow_run_id=workflow_run.workflow_run_id,+ organization_id=organization_id,+ )+ if existing_retry_run_id:+ return existing_retry_run_id++ try:+ workflow = await app.WORKFLOW_SERVICE.get_workflow(+ workflow_id=workflow_run.workflow_id,+ organization_id=organization_id,+ )+ except WorkflowNotFound:+ return None+ if workflow is None:+ return None++ parameter_tuples = await app.DATABASE.workflow_runs.get_workflow_run_parameters(+ workflow_run_id=workflow_run.workflow_run_id,+ )+ retry_parameters = {+ workflow_parameter.key: workflow_run_parameter.value+ for workflow_parameter, workflow_run_parameter in parameter_tuples+ }++ organization = await app.DATABASE.organizations.get_organization(organization_id)+ if organization is None:+ return None++ from skyvern.services.workflow_service import run_workflow, workflow_request_body_from_existing_run++ run_metadata = await _reload_user_run_metadata(workflow_run, organization_id)+ workflow_request = workflow_request_body_from_existing_run(+ workflow_run=workflow_run,+ parameters=retry_parameters,+ run_metadata=run_metadata,+ )+ # The whole point of the fallback: the retry runs as a pure agent, never code.+ workflow_request.run_with = AGENT_RUN_WITH+ # Shed every handle to the failed run's browser so the agent starts from a clean session instead+ # of reconnecting to the code run's half-filled page / authenticated state.+ workflow_request.browser_session_id = None+ workflow_request.browser_profile_id = None+ workflow_request.browser_address = None+ workflow_request.cdp_connect_headers = None+ workflow_request.extra_http_headers = app.AGENT_FUNCTION.strip_proxy_session_extra_http_headers(+ workflow_request.extra_http_headers+ )++ next_attempt = (workflow_run.fallback_attempt or 0) + 1+ try:+ # A fresh context keeps run creation (and the background execution task it spawns, which+ # inherits contextvars) from attributing the retry to the failed run.+ with skyvern_context.scoped(SkyvernContext()):+ retry_run = await run_workflow(+ workflow_id=workflow_run.workflow_permanent_id,+ organization=organization,+ workflow_request=workflow_request,+ template=False,+ version=workflow.version,+ trigger_type=workflow_run.trigger_type,+ workflow_schedule_id=workflow_run.workflow_schedule_id,+ retried_from_workflow_run_id=workflow_run.workflow_run_id,+ fallback_attempt=next_attempt,+ ignore_inherited_workflow_system_prompt=workflow_run.ignore_inherited_workflow_system_prompt,+ )+ except IntegrityError as exc:+ error_text = str(exc.orig) if exc.orig is not None else str(exc)+ if (+ "ix_workflow_runs_retried_from_workflow_run_id" not in error_text+ and "retried_from_workflow_run_id" not in error_text+ ):+ raise+ existing_retry_run_id = await app.DATABASE.workflow_runs.get_workflow_run_retried_by(+ workflow_run_id=workflow_run.workflow_run_id,+ organization_id=organization_id,+ )+ if not existing_retry_run_id:+ raise+ return existing_retry_run_id++ LOG.info(+ "Started code-mode fallback retry",+ workflow_run_id=workflow_run.workflow_run_id,+ new_run_id=retry_run.workflow_run_id,+ fallback_attempt=next_attempt,+ )+ return retry_run.workflow_run_iddiff --git a/skyvern/forge/sdk/workflow/service.py b/skyvern/forge/sdk/workflow/service.pyindex d2eb87c1e..918f7ce83 100644--- a/skyvern/forge/sdk/workflow/service.py+++ b/skyvern/forge/sdk/workflow/service.py@@ -83,6 +83,7 @@ from skyvern.forge.sdk.workflow.browser_profile_key import (build_workflow_browser_session_storage_key,render_browser_profile_key,)+from skyvern.forge.sdk.workflow.code_mode_fallback import maybe_start_code_mode_fallback_retryfrom skyvern.forge.sdk.workflow.credential_fallback import (VALID_FALLBACK_TRIGGERS,maybe_start_credential_fallback_retry,@@ -737,9 +738,12 @@ class WorkflowService:)@staticmethod- async def _start_credential_fallback_retry_best_effort(workflow_run: WorkflowRun) -> None:+ async def _start_run_fallback_retries_best_effort(workflow_run: WorkflowRun) -> None:+ # Credential fallback first: it targets auth failures specifically. If it claims the+ # single retry slot (retried_from unique index), code-mode fallback no-ops. Run them+ # sequentially in one task so they can't race to create the retry.try:- await maybe_start_credential_fallback_retry(workflow_run, workflow_run.organization_id)+ retry_run_id = await maybe_start_credential_fallback_retry(workflow_run, workflow_run.organization_id)except Exception:LOG.warning("Credential fallback retry hook failed",@@ -747,9 +751,21 @@ class WorkflowService:organization_id=workflow_run.organization_id,exc_info=True,)+ retry_run_id = None+ if retry_run_id:+ return+ try:+ await maybe_start_code_mode_fallback_retry(workflow_run, workflow_run.organization_id)+ except Exception:+ LOG.warning(+ "Code-mode fallback retry hook failed",+ workflow_run_id=workflow_run.workflow_run_id,+ organization_id=workflow_run.organization_id,+ exc_info=True,+ )- def _schedule_credential_fallback_retry(self, workflow_run: WorkflowRun) -> None:- task = asyncio.create_task(self._start_credential_fallback_retry_best_effort(workflow_run))+ def _schedule_run_fallback_retries(self, workflow_run: WorkflowRun) -> None:+ task = asyncio.create_task(self._start_run_fallback_retries_best_effort(workflow_run))self._background_tasks.add(task)task.add_done_callback(self._background_tasks.discard)@@ -7465,13 +7481,14 @@ class WorkflowService:for child_workflow_run_id in child_workflow_run_ids:app.WORKFLOW_CONTEXT_MANAGER.remove_workflow_run_context(child_workflow_run_id)- # Schedule the credential fallback retry only after this run's cleanup (artifact/video- # persistence, browser/session write-back, webhook) has run, so the replacement run- # cannot overlap it and race browser-profile/session writes. In the finally wrapping the- # whole cleanup sequence — not after it — so an exception in any cleanup step can't skip- # it; scheduling was removed from the status markers, which fired before cleanup and for- # cascade/reaper failures that never reach cleanup. A no-op for non-eligible runs.- self._schedule_credential_fallback_retry(workflow_run)+ # Schedule the fallback retries (credential, then code→agent) only after this run's+ # cleanup (artifact/video persistence, browser/session write-back, webhook) has run, so+ # the replacement run cannot overlap it and race browser-profile/session writes. In the+ # finally wrapping the whole cleanup sequence — not after it — so an exception in any+ # cleanup step can't skip it; scheduling was removed from the status markers, which fired+ # before cleanup and for cascade/reaper failures that never reach cleanup. A no-op for+ # non-eligible runs.+ self._schedule_run_fallback_retries(workflow_run)async def prepare_workflow_webhook(self,@@ -9162,12 +9179,26 @@ class WorkflowService:"""Whether this run should attempt cached-script execution.Priority: run-level run_with > workflow-level run_with. Intended-code- runs are then passed through the app-level code-mode gate.+ runs are then passed through the app-level code-mode gate. Runs not+ already intended as code may be upgraded by a flag-gated rollout;+ an upgraded run has no matching script for a non-ATS workflow and so+ degrades cleanly to agent per block."""if workflow_run.run_with is not None:intended_code = workflow_run.run_with == "code"else:intended_code = workflow.run_with == "code"+ # A fallback retry has already made a deliberate execution-mode choice — in particular the+ # code->agent fallback sets run_with=agent precisely so the retry does NOT run the script+ # that just failed. The rollout must not re-upgrade it back to code, or the fallback would be+ # a no-op for exactly the rollout population. Gate on the retry marker rather than+ # `run_with is None`: normal runs inherit run_with="agent" from the workflow, so keying on+ # None would disable the rollout for every ordinary run.+ if not intended_code and workflow_run.retried_from_workflow_run_id is None:+ intended_code = await app.AGENT_FUNCTION.should_upgrade_to_code_mode(+ workflow=workflow,+ workflow_run=workflow_run,+ )if not intended_code:return Falsereturn await app.AGENT_FUNCTION.should_keep_code_mode_for_workflow_run(diff --git a/tests/unit/test_code_mode_fallback.py b/tests/unit/test_code_mode_fallback.pynew file mode 100644index 000000000..c539ce668--- /dev/null+++ b/tests/unit/test_code_mode_fallback.py@@ -0,0 +1,168 @@+from __future__ import annotations++from datetime import datetime, timezone+from types import SimpleNamespace+from unittest.mock import AsyncMock, MagicMock, patch++import pytest++from skyvern.forge.sdk.db.enums import WorkflowRunTriggerType+from skyvern.forge.sdk.workflow.code_mode_fallback import (+ _trigger_matches,+ maybe_start_code_mode_fallback_retry,+)+from skyvern.forge.sdk.workflow.models.workflow import WorkflowRun, WorkflowRunStatus++MODULE = "skyvern.forge.sdk.workflow.code_mode_fallback"+++def _workflow_run(+ *,+ status: WorkflowRunStatus = WorkflowRunStatus.failed,+ run_with: str | None = "code",+ fallback_attempt: int | None = None,+ parent_workflow_run_id: str | None = None,+ debug_session_id: str | None = None,+ copilot_session_id: str | None = None,+ retried_from_workflow_run_id: str | None = None,+) -> WorkflowRun:+ now = datetime.now(timezone.utc)+ return WorkflowRun(+ workflow_run_id="wr_failed",+ workflow_id="wf_test",+ workflow_permanent_id="wpid_test",+ organization_id="org_test",+ status=status,+ run_with=run_with,+ failure_reason="element not found",+ fallback_attempt=fallback_attempt,+ parent_workflow_run_id=parent_workflow_run_id,+ debug_session_id=debug_session_id,+ copilot_session_id=copilot_session_id,+ retried_from_workflow_run_id=retried_from_workflow_run_id,+ trigger_type=WorkflowRunTriggerType.api,+ created_at=now,+ modified_at=now,+ )+++def _mock_app(*, flag_enabled: bool = True, existing_retry: str | None = None, block_scoped: bool = False) -> MagicMock:+ mock_app = MagicMock()+ provider = SimpleNamespace(is_feature_enabled_cached=AsyncMock(return_value=flag_enabled))+ mock_app.EXPERIMENTATION_PROVIDER = provider+ mock_app.AGENT_FUNCTION.is_block_scoped_workflow_run = AsyncMock(return_value=block_scoped)+ mock_app.AGENT_FUNCTION.strip_proxy_session_extra_http_headers = MagicMock(return_value=None)+ mock_app.DATABASE.debug.has_block_run_for_workflow_run = AsyncMock(return_value=False)+ mock_app.DATABASE.workflow_runs.get_workflow_run_retried_by = AsyncMock(return_value=existing_retry)+ mock_app.DATABASE.workflow_runs.get_workflow_run_parameters = AsyncMock(return_value=[])+ mock_app.DATABASE.organizations.get_organization = AsyncMock(+ return_value=SimpleNamespace(organization_id="org_test")+ )+ mock_app.DATABASE.tags.get_active_grouped_tags_for_run = AsyncMock(return_value={})+ mock_app.WORKFLOW_SERVICE.get_workflow = AsyncMock(+ return_value=SimpleNamespace(version=7, workflow_permanent_id="wpid_test")+ )+ return mock_app+++ "status,run_with,expected",+ [+ (WorkflowRunStatus.failed, "code", True),+ (WorkflowRunStatus.terminated, "code", True),+ (WorkflowRunStatus.timed_out, "code", False),+ (WorkflowRunStatus.completed, "code", False),+ (WorkflowRunStatus.failed, "agent", False),+ (WorkflowRunStatus.failed, None, False),+ ],+)+def test_trigger_matches(status: WorkflowRunStatus, run_with: str | None, expected: bool) -> None:+ assert _trigger_matches(_workflow_run(status=status, run_with=run_with)) is expected+++async def test_spawns_agent_retry_for_failed_code_run() -> None:+ workflow_run = _workflow_run()+ request = MagicMock()+ with (+ patch(f"{MODULE}.app", _mock_app()),+ patch(+ "skyvern.services.workflow_service.workflow_request_body_from_existing_run",+ MagicMock(return_value=request),+ ),+ patch(+ "skyvern.services.workflow_service.run_workflow",+ AsyncMock(return_value=SimpleNamespace(workflow_run_id="wr_retry")),+ ) as run_workflow,+ ):+ result = await maybe_start_code_mode_fallback_retry(workflow_run, "org_test")++ assert result == "wr_retry"+ # The retry must run as a pure agent and drop the code run's browser handles.+ assert request.run_with == "agent"+ assert request.browser_session_id is None+ assert request.browser_profile_id is None+ _, kwargs = run_workflow.call_args+ assert kwargs["retried_from_workflow_run_id"] == "wr_failed"+ assert kwargs["fallback_attempt"] == 1+++async def test_no_retry_when_flag_disabled() -> None:+ with (+ patch(f"{MODULE}.app", _mock_app(flag_enabled=False)),+ patch("skyvern.services.workflow_service.run_workflow", AsyncMock()) as run_workflow,+ ):+ result = await maybe_start_code_mode_fallback_retry(_workflow_run(), "org_test")++ assert result is None+ run_workflow.assert_not_called()+++async def test_returns_existing_retry_without_spawning() -> None:+ with (+ patch(f"{MODULE}.app", _mock_app(existing_retry="wr_prior")),+ patch("skyvern.services.workflow_service.run_workflow", AsyncMock()) as run_workflow,+ ):+ result = await maybe_start_code_mode_fallback_retry(_workflow_run(), "org_test")++ assert result == "wr_prior"+ run_workflow.assert_not_called()+++async def test_no_retry_when_block_scoped() -> None:+ with (+ patch(f"{MODULE}.app", _mock_app(block_scoped=True)),+ patch("skyvern.services.workflow_service.run_workflow", AsyncMock()) as run_workflow,+ ):+ result = await maybe_start_code_mode_fallback_retry(_workflow_run(), "org_test")++ assert result is None+ run_workflow.assert_not_called()+++ "kwargs",+ [+ {"run_with": "agent"},+ {"run_with": None},+ {"status": WorkflowRunStatus.timed_out},+ {"fallback_attempt": 1},+ {"retried_from_workflow_run_id": "wr_orig"},+ {"parent_workflow_run_id": "wr_parent"},+ {"debug_session_id": "dbg_1"},+ {"copilot_session_id": "cop_1"},+ ],+)+async def test_ineligible_runs_never_spawn(kwargs: dict) -> None:+ with (+ patch(f"{MODULE}.app", _mock_app()),+ patch("skyvern.services.workflow_service.run_workflow", AsyncMock()) as run_workflow,+ ):+ result = await maybe_start_code_mode_fallback_retry(_workflow_run(**kwargs), "org_test")++ assert result is None+ run_workflow.assert_not_called()diff --git a/tests/unit/test_credential_fallback.py b/tests/unit/test_credential_fallback.pyindex 8d19986b8..f3c640a56 100644--- a/tests/unit/test_credential_fallback.py+++ b/tests/unit/test_credential_fallback.py@@ -846,7 +846,7 @@ async def test_mark_workflow_run_as_failed_does_not_schedule_before_cleanup() ->service = WorkflowService()workflow_run = _workflow_run(failure_category=[{"category": "AUTH_FAILURE"}])service._update_workflow_run_status = AsyncMock(return_value=workflow_run) # type: ignore[method-assign]- service._schedule_credential_fallback_retry = MagicMock() # type: ignore[method-assign]+ service._schedule_run_fallback_retries = MagicMock() # type: ignore[method-assign]result = await service.mark_workflow_run_as_failed(workflow_run_id="wr_failed",@@ -855,7 +855,7 @@ async def test_mark_workflow_run_as_failed_does_not_schedule_before_cleanup() ->)assert result == workflow_run- service._schedule_credential_fallback_retry.assert_not_called()+ service._schedule_run_fallback_retries.assert_not_called()@pytest.mark.asyncio@@ -866,7 +866,7 @@ async def test_mark_workflow_run_as_terminated_does_not_schedule_before_cleanup(failure_category=[{"category": "AUTH_FAILURE"}],)service._update_workflow_run_status = AsyncMock(return_value=workflow_run) # type: ignore[method-assign]- service._schedule_credential_fallback_retry = MagicMock() # type: ignore[method-assign]+ service._schedule_run_fallback_retries = MagicMock() # type: ignore[method-assign]result = await service.mark_workflow_run_as_terminated(workflow_run_id="wr_failed",@@ -875,7 +875,7 @@ async def test_mark_workflow_run_as_terminated_does_not_schedule_before_cleanup()assert result == workflow_run- service._schedule_credential_fallback_retry.assert_not_called()+ service._schedule_run_fallback_retries.assert_not_called()@pytest.mark.asyncio@@ -883,7 +883,7 @@ async def test_clean_up_workflow_schedules_credential_fallback_retry() -> None:service = WorkflowService()workflow_run = _workflow_run(failure_category=[{"category": "AUTH_FAILURE"}])workflow = _workflow([])- service._schedule_credential_fallback_retry = MagicMock() # type: ignore[method-assign]+ service._schedule_run_fallback_retries = MagicMock() # type: ignore[method-assign]browser_cleanup_result = SimpleNamespace(browser_state=None,tasks=[],@@ -909,7 +909,7 @@ async def test_clean_up_workflow_schedules_credential_fallback_retry() -> None:browser_cleanup_result=browser_cleanup_result, # type: ignore[arg-type])- service._schedule_credential_fallback_retry.assert_called_once_with(workflow_run)+ service._schedule_run_fallback_retries.assert_called_once_with(workflow_run)@pytest.mark.asyncio@@ -919,7 +919,7 @@ async def test_clean_up_workflow_schedules_retry_even_when_webhook_raises() -> Nservice = WorkflowService()workflow_run = _workflow_run(failure_category=[{"category": "AUTH_FAILURE"}])workflow = _workflow([])- service._schedule_credential_fallback_retry = MagicMock() # type: ignore[method-assign]+ service._schedule_run_fallback_retries = MagicMock() # type: ignore[method-assign]service.execute_workflow_webhook = AsyncMock(side_effect=RuntimeError("webhook prep failed")) # type: ignore[method-assign]browser_cleanup_result = SimpleNamespace(browser_state=None,@@ -947,7 +947,7 @@ async def test_clean_up_workflow_schedules_retry_even_when_webhook_raises() -> Nbrowser_cleanup_result=browser_cleanup_result, # type: ignore[arg-type])- service._schedule_credential_fallback_retry.assert_called_once_with(workflow_run)+ service._schedule_run_fallback_retries.assert_called_once_with(workflow_run)@pytest.mark.asyncio@@ -957,7 +957,7 @@ async def test_clean_up_workflow_schedules_retry_when_earlier_cleanup_step_raiseservice = WorkflowService()workflow_run = _workflow_run(failure_category=[{"category": "AUTH_FAILURE"}])workflow = _workflow([])- service._schedule_credential_fallback_retry = MagicMock() # type: ignore[method-assign]+ service._schedule_run_fallback_retries = MagicMock() # type: ignore[method-assign]browser_cleanup_result = SimpleNamespace(browser_state=None,tasks=[],@@ -984,7 +984,7 @@ async def test_clean_up_workflow_schedules_retry_when_earlier_cleanup_step_raisebrowser_cleanup_result=browser_cleanup_result, # type: ignore[arg-type])- service._schedule_credential_fallback_retry.assert_called_once_with(workflow_run)+ service._schedule_run_fallback_retries.assert_called_once_with(workflow_run)@pytest.mark.asyncio@@ -995,4 +995,4 @@ async def test_fallback_hook_exception_does_not_raise() -> None:"skyvern.forge.sdk.workflow.service.maybe_start_credential_fallback_retry",AsyncMock(side_effect=RuntimeError("boom")),):- await WorkflowService._start_credential_fallback_retry_best_effort(workflow_run)+ await WorkflowService._start_run_fallback_retries_best_effort(workflow_run)diff --git a/tests/unit/test_should_run_script.py b/tests/unit/test_should_run_script.pyindex f42573a71..9d712547e 100644--- a/tests/unit/test_should_run_script.py+++ b/tests/unit/test_should_run_script.py@@ -45,7 +45,11 @@ def _make_workflow()-def _make_run(run_with: str | None = "agent", workflow_run_id: str = "wr_test") -> WorkflowRun:+def _make_run(+ run_with: str | None = "agent",+ workflow_run_id: str = "wr_test",+ retried_from_workflow_run_id: str | None = None,+) -> WorkflowRun:return WorkflowRun(workflow_run_id=workflow_run_id,workflow_id="wf_test",@@ -53,6 +57,7 @@ def _make_run(run_with: str | None = "agent", workflow_run_id: str = "wr_test")organization_id="org_test",status=WorkflowRunStatus.running,run_with=run_with,+ retried_from_workflow_run_id=retried_from_workflow_run_id,created_at=datetime.now(timezone.utc),modified_at=datetime.now(timezone.utc),)@@ -230,6 +235,83 @@ class TestCodeModeGate:assert len(agent_function.calls) == 1+class _UpgradingAgentFunction(AgentFunction):+ def __init__(self, upgrade: bool, keep_code: bool = True) -> None:+ self.upgrade = upgrade+ self.keep_code = keep_code+ self.upgrade_calls: list[tuple[str, str, str]] = []++ async def should_upgrade_to_code_mode(self, *, workflow: Workflow, workflow_run: WorkflowRun) -> bool:+ self.upgrade_calls.append(+ (workflow_run.workflow_run_id, workflow_run.organization_id, workflow.workflow_permanent_id)+ )+ return self.upgrade++ async def should_keep_code_mode_for_workflow_run(self, *, workflow: Workflow, workflow_run: WorkflowRun) -> bool:+ return self.keep_code+++class TestCodeModeUpgrade:+ """A flag-gated rollout can upgrade an agent-intended run to code."""++ @pytest.mark.asyncio+ async def test_upgrade_flips_agent_to_code(self, monkeypatch):+ agent_function = _UpgradingAgentFunction(upgrade=True, keep_code=True)+ monkeypatch.setattr(app, "AGENT_FUNCTION", agent_function)+ from skyvern.forge.sdk.workflow.service import WorkflowService++ wf = _make_workflow(run_with="agent")+ wr = _make_run(run_with="agent")+ assert await WorkflowService().should_run_script(wf, wr) is True+ assert agent_function.upgrade_calls == [("wr_test", "org_test", "wpid_test")]++ @pytest.mark.asyncio+ async def test_upgrade_disabled_stays_agent(self, monkeypatch):+ agent_function = _UpgradingAgentFunction(upgrade=False)+ monkeypatch.setattr(app, "AGENT_FUNCTION", agent_function)+ from skyvern.forge.sdk.workflow.service import WorkflowService++ wf = _make_workflow(run_with="agent")+ wr = _make_run(run_with="agent")+ assert await WorkflowService().should_run_script(wf, wr) is False++ @pytest.mark.asyncio+ async def test_upgraded_run_still_passes_keep_code_gate(self, monkeypatch):+ """Upgrade True but keep-code gate False → still agent (gate can veto)."""+ agent_function = _UpgradingAgentFunction(upgrade=True, keep_code=False)+ monkeypatch.setattr(app, "AGENT_FUNCTION", agent_function)+ from skyvern.forge.sdk.workflow.service import WorkflowService++ wf = _make_workflow(run_with="agent")+ wr = _make_run(run_with="agent")+ assert await WorkflowService().should_run_script(wf, wr) is False++ @pytest.mark.asyncio+ async def test_upgrade_not_consulted_when_already_code(self, monkeypatch):+ agent_function = _UpgradingAgentFunction(upgrade=False)+ monkeypatch.setattr(app, "AGENT_FUNCTION", agent_function)+ from skyvern.forge.sdk.workflow.service import WorkflowService++ wf = _make_workflow(run_with="code")+ wr = _make_run(run_with="code")+ assert await WorkflowService().should_run_script(wf, wr) is True+ assert agent_function.upgrade_calls == []++ @pytest.mark.asyncio+ async def test_fallback_retry_never_re_upgraded(self, monkeypatch):+ # A code->agent fallback retry (run_with=agent, retried_from set) must never be re-upgraded+ # to code, or the rollout would re-run the very script that just failed and the fallback+ # would be a no-op for exactly the rollout population.+ agent_function = _UpgradingAgentFunction(upgrade=True, keep_code=True)+ monkeypatch.setattr(app, "AGENT_FUNCTION", agent_function)+ from skyvern.forge.sdk.workflow.service import WorkflowService++ wf = _make_workflow(run_with="agent")+ wr = _make_run(run_with="agent", retried_from_workflow_run_id="wr_orig")+ assert await WorkflowService().should_run_script(wf, wr) is False+ assert agent_function.upgrade_calls == []++class TestScriptReviewerGate:"""The script reviewer should only fire when the script was actually executed.