Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
Revert "Replace the Workday static script (deterministic) + flag-gated code-mode rollout" (#7620)
Cindy Li2f29ec12dd8b
diff --git a/skyvern/forge/agent_functions.py b/skyvern/forge/agent_functions.pyindex 7297082c3..17fa33e35 100644--- a/skyvern/forge/agent_functions.py+++ b/skyvern/forge/agent_functions.py@@ -802,14 +802,6 @@ class AgentFunction:) -> bool:return True- async def should_upgrade_to_code_mode(- self,- *,- workflow: "Workflow",- workflow_run: "WorkflowRun",- ) -> bool:- return False-async def resolve_mcp_code_only_mode(self,organization_id: str | None,diff --git a/skyvern/forge/sdk/workflow/code_mode_fallback.py b/skyvern/forge/sdk/workflow/code_mode_fallback.pydeleted file mode 100644index 99272c8ac..000000000--- a/skyvern/forge/sdk/workflow/code_mode_fallback.py+++ /dev/null@@ -1,217 +0,0 @@-from __future__ import annotations--import structlog-from sqlalchemy.exc import IntegrityError--from skyvern.exceptions import WorkflowNotFound-from skyvern.forge import app-from skyvern.forge.sdk.core import skyvern_context-from skyvern.forge.sdk.core.skyvern_context import SkyvernContext-from skyvern.forge.sdk.workflow.models.validators import is_reserved_tag_key-from skyvern.forge.sdk.workflow.models.workflow import WorkflowRun, WorkflowRunStatus--LOG = structlog.get_logger()--CODE_MODE_FALLBACK_RETRY_FLAG = "CODE_MODE_FALLBACK_RETRY"-CODE_RUN_WITH = "code"-AGENT_RUN_WITH = "agent"-# A code-mode run that failed/terminated retries once as a pure-agent run. timed_out is excluded:-# re-running the full (slower) agent path on an already-long run is rarely what the caller wants.-#-# Double-apply risk (accepted, flag-gated): the retry is a fresh full-workflow run, so a run that-# submitted an application and then failed on a later block would submit again. A run that actually-# completed never reaches here (only failed/terminated retry), which covers the common case; the-# submitted-then-errored edge is real, so the feature stays off by default (CODE_MODE_FALLBACK_RETRY)-# and is enabled per-org only with explicit owner sign-off that the target tolerates a possible-# duplicate submission.-RETRYABLE_STATUSES = frozenset({WorkflowRunStatus.failed, WorkflowRunStatus.terminated})---async def _retry_enabled_for_organization(organization_id: str, workflow_run: WorkflowRun) -> bool:- """Gate the auto-retry rollout. Fails closed: a retry run costs credits and re-fires webhooks.-- distinct_id = organization_id (deliberately NOT run-level), so an org is fully in-or-out- rather than retrying on a random subset of its runs. Mirrors credential_fallback.- """- provider = getattr(app, "EXPERIMENTATION_PROVIDER", None)- if not provider:- return False- try:- return bool(- await provider.is_feature_enabled_cached(- CODE_MODE_FALLBACK_RETRY_FLAG,- organization_id,- properties={- "organization_id": organization_id,- "workflow_permanent_id": workflow_run.workflow_permanent_id,- },- )- )- except Exception:- LOG.warning(- "code_mode_fallback.flag_error",- workflow_run_id=workflow_run.workflow_run_id,- organization_id=organization_id,- exc_info=True,- )- return False---def _trigger_matches(workflow_run: WorkflowRun) -> bool:- """A code-mode run in a retryable terminal state is eligible; anything else is not."""- if workflow_run.status not in RETRYABLE_STATUSES:- return False- return (workflow_run.run_with or AGENT_RUN_WITH) == CODE_RUN_WITH---async def _reload_user_run_metadata(workflow_run: WorkflowRun, organization_id: str) -> dict[str, str] | None:- """Reload the failed run's user-writable tags so the retry keeps them. Reserved keys are dropped —- they are re-derived per run. Mirrors credential_fallback / the manual retry path."""- try:- grouped = await app.DATABASE.tags.get_active_grouped_tags_for_run(- workflow_run_id=workflow_run.workflow_run_id,- organization_id=organization_id,- )- except Exception:- LOG.warning(- "Failed to reload run metadata for code-mode fallback retry; continuing without it",- workflow_run_id=workflow_run.workflow_run_id,- exc_info=True,- )- return None- if not grouped:- return None- return {key: value for key, value in grouped.items() if not is_reserved_tag_key(key)} or None---async def maybe_start_code_mode_fallback_retry(workflow_run: WorkflowRun, organization_id: str) -> str | None:- try:- return await _maybe_start_code_mode_fallback_retry(workflow_run, organization_id)- except Exception:- LOG.warning(- "Failed to start code-mode fallback retry",- workflow_run_id=workflow_run.workflow_run_id,- organization_id=organization_id,- exc_info=True,- )- return None---async def _maybe_start_code_mode_fallback_retry(workflow_run: WorkflowRun, organization_id: str) -> str | None:- # Scheduled from clean_up_workflow, which runs for every terminal status. Bail on ineligible runs- # before the flag check and DB reads so the success path stays cheap.- if not _trigger_matches(workflow_run):- return None-- # Never retry a retry: an agent attempt that fails is terminal, and a run that is already a- # fallback retry (credential or code) must not spawn another. This also bounds the chain.- if workflow_run.retried_from_workflow_run_id or workflow_run.fallback_attempt:- return None-- # Block-run / copilot-test / debug runs supply block_labels directly and must not spawn a- # detached full-workflow run (real credits, real side effects) outside the test.- if workflow_run.parent_workflow_run_id or workflow_run.debug_session_id or workflow_run.copilot_session_id:- return None-- if not await _retry_enabled_for_organization(organization_id, workflow_run):- return None-- is_block_scoped_run = await app.AGENT_FUNCTION.is_block_scoped_workflow_run(workflow_run)- if not is_block_scoped_run:- is_block_scoped_run = await app.DATABASE.debug.has_block_run_for_workflow_run(- organization_id=organization_id,- workflow_run_id=workflow_run.workflow_run_id,- )- if is_block_scoped_run:- return None-- # Idempotency: the retried_from_workflow_run_id unique index allows one retry per run, shared- # with credential_fallback — so if credential fallback already claimed the slot, no-op.- existing_retry_run_id = await app.DATABASE.workflow_runs.get_workflow_run_retried_by(- workflow_run_id=workflow_run.workflow_run_id,- organization_id=organization_id,- )- if existing_retry_run_id:- return existing_retry_run_id-- try:- workflow = await app.WORKFLOW_SERVICE.get_workflow(- workflow_id=workflow_run.workflow_id,- organization_id=organization_id,- )- except WorkflowNotFound:- return None- if workflow is None:- return None-- parameter_tuples = await app.DATABASE.workflow_runs.get_workflow_run_parameters(- workflow_run_id=workflow_run.workflow_run_id,- )- retry_parameters = {- workflow_parameter.key: workflow_run_parameter.value- for workflow_parameter, workflow_run_parameter in parameter_tuples- }-- organization = await app.DATABASE.organizations.get_organization(organization_id)- if organization is None:- return None-- from skyvern.services.workflow_service import run_workflow, workflow_request_body_from_existing_run-- run_metadata = await _reload_user_run_metadata(workflow_run, organization_id)- workflow_request = workflow_request_body_from_existing_run(- workflow_run=workflow_run,- parameters=retry_parameters,- run_metadata=run_metadata,- )- # The whole point of the fallback: the retry runs as a pure agent, never code.- workflow_request.run_with = AGENT_RUN_WITH- # Shed every handle to the failed run's browser so the agent starts from a clean session instead- # of reconnecting to the code run's half-filled page / authenticated state.- workflow_request.browser_session_id = None- workflow_request.browser_profile_id = None- workflow_request.browser_address = None- workflow_request.cdp_connect_headers = None- workflow_request.extra_http_headers = app.AGENT_FUNCTION.strip_proxy_session_extra_http_headers(- workflow_request.extra_http_headers- )-- next_attempt = (workflow_run.fallback_attempt or 0) + 1- try:- # A fresh context keeps run creation (and the background execution task it spawns, which- # inherits contextvars) from attributing the retry to the failed run.- with skyvern_context.scoped(SkyvernContext()):- retry_run = await run_workflow(- workflow_id=workflow_run.workflow_permanent_id,- organization=organization,- workflow_request=workflow_request,- template=False,- version=workflow.version,- trigger_type=workflow_run.trigger_type,- workflow_schedule_id=workflow_run.workflow_schedule_id,- retried_from_workflow_run_id=workflow_run.workflow_run_id,- fallback_attempt=next_attempt,- ignore_inherited_workflow_system_prompt=workflow_run.ignore_inherited_workflow_system_prompt,- )- except IntegrityError as exc:- error_text = str(exc.orig) if exc.orig is not None else str(exc)- if (- "ix_workflow_runs_retried_from_workflow_run_id" not in error_text- and "retried_from_workflow_run_id" not in error_text- ):- raise- existing_retry_run_id = await app.DATABASE.workflow_runs.get_workflow_run_retried_by(- workflow_run_id=workflow_run.workflow_run_id,- organization_id=organization_id,- )- if not existing_retry_run_id:- raise- return existing_retry_run_id-- LOG.info(- "Started code-mode fallback retry",- workflow_run_id=workflow_run.workflow_run_id,- new_run_id=retry_run.workflow_run_id,- fallback_attempt=next_attempt,- )- return retry_run.workflow_run_iddiff --git a/skyvern/forge/sdk/workflow/service.py b/skyvern/forge/sdk/workflow/service.pyindex 918f7ce83..d2eb87c1e 100644--- a/skyvern/forge/sdk/workflow/service.py+++ b/skyvern/forge/sdk/workflow/service.py@@ -83,7 +83,6 @@ from skyvern.forge.sdk.workflow.browser_profile_key import (build_workflow_browser_session_storage_key,render_browser_profile_key,)-from skyvern.forge.sdk.workflow.code_mode_fallback import maybe_start_code_mode_fallback_retryfrom skyvern.forge.sdk.workflow.credential_fallback import (VALID_FALLBACK_TRIGGERS,maybe_start_credential_fallback_retry,@@ -738,12 +737,9 @@ class WorkflowService:)@staticmethod- async def _start_run_fallback_retries_best_effort(workflow_run: WorkflowRun) -> None:- # Credential fallback first: it targets auth failures specifically. If it claims the- # single retry slot (retried_from unique index), code-mode fallback no-ops. Run them- # sequentially in one task so they can't race to create the retry.+ async def _start_credential_fallback_retry_best_effort(workflow_run: WorkflowRun) -> None:try:- retry_run_id = await maybe_start_credential_fallback_retry(workflow_run, workflow_run.organization_id)+ await maybe_start_credential_fallback_retry(workflow_run, workflow_run.organization_id)except Exception:LOG.warning("Credential fallback retry hook failed",@@ -751,21 +747,9 @@ class WorkflowService:organization_id=workflow_run.organization_id,exc_info=True,)- retry_run_id = None- if retry_run_id:- return- try:- await maybe_start_code_mode_fallback_retry(workflow_run, workflow_run.organization_id)- except Exception:- LOG.warning(- "Code-mode fallback retry hook failed",- workflow_run_id=workflow_run.workflow_run_id,- organization_id=workflow_run.organization_id,- exc_info=True,- )- def _schedule_run_fallback_retries(self, workflow_run: WorkflowRun) -> None:- task = asyncio.create_task(self._start_run_fallback_retries_best_effort(workflow_run))+ def _schedule_credential_fallback_retry(self, workflow_run: WorkflowRun) -> None:+ task = asyncio.create_task(self._start_credential_fallback_retry_best_effort(workflow_run))self._background_tasks.add(task)task.add_done_callback(self._background_tasks.discard)@@ -7481,14 +7465,13 @@ class WorkflowService:for child_workflow_run_id in child_workflow_run_ids:app.WORKFLOW_CONTEXT_MANAGER.remove_workflow_run_context(child_workflow_run_id)- # Schedule the fallback retries (credential, then code→agent) only after this run's- # cleanup (artifact/video persistence, browser/session write-back, webhook) has run, so- # the replacement run cannot overlap it and race browser-profile/session writes. In the- # finally wrapping the whole cleanup sequence — not after it — so an exception in any- # cleanup step can't skip it; scheduling was removed from the status markers, which fired- # before cleanup and for cascade/reaper failures that never reach cleanup. A no-op for- # non-eligible runs.- self._schedule_run_fallback_retries(workflow_run)+ # Schedule the credential fallback retry only after this run's cleanup (artifact/video+ # persistence, browser/session write-back, webhook) has run, so the replacement run+ # cannot overlap it and race browser-profile/session writes. In the finally wrapping the+ # whole cleanup sequence — not after it — so an exception in any cleanup step can't skip+ # it; scheduling was removed from the status markers, which fired before cleanup and for+ # cascade/reaper failures that never reach cleanup. A no-op for non-eligible runs.+ self._schedule_credential_fallback_retry(workflow_run)async def prepare_workflow_webhook(self,@@ -9179,26 +9162,12 @@ class WorkflowService:"""Whether this run should attempt cached-script execution.Priority: run-level run_with > workflow-level run_with. Intended-code- runs are then passed through the app-level code-mode gate. Runs not- already intended as code may be upgraded by a flag-gated rollout;- an upgraded run has no matching script for a non-ATS workflow and so- degrades cleanly to agent per block.+ runs are then passed through the app-level code-mode gate."""if workflow_run.run_with is not None:intended_code = workflow_run.run_with == "code"else:intended_code = workflow.run_with == "code"- # A fallback retry has already made a deliberate execution-mode choice — in particular the- # code->agent fallback sets run_with=agent precisely so the retry does NOT run the script- # that just failed. The rollout must not re-upgrade it back to code, or the fallback would be- # a no-op for exactly the rollout population. Gate on the retry marker rather than- # `run_with is None`: normal runs inherit run_with="agent" from the workflow, so keying on- # None would disable the rollout for every ordinary run.- if not intended_code and workflow_run.retried_from_workflow_run_id is None:- intended_code = await app.AGENT_FUNCTION.should_upgrade_to_code_mode(- workflow=workflow,- workflow_run=workflow_run,- )if not intended_code:return Falsereturn await app.AGENT_FUNCTION.should_keep_code_mode_for_workflow_run(diff --git a/tests/unit/test_code_mode_fallback.py b/tests/unit/test_code_mode_fallback.pydeleted file mode 100644index c539ce668..000000000--- a/tests/unit/test_code_mode_fallback.py+++ /dev/null@@ -1,168 +0,0 @@-from __future__ import annotations--from datetime import datetime, timezone-from types import SimpleNamespace-from unittest.mock import AsyncMock, MagicMock, patch--import pytest--from skyvern.forge.sdk.db.enums import WorkflowRunTriggerType-from skyvern.forge.sdk.workflow.code_mode_fallback import (- _trigger_matches,- maybe_start_code_mode_fallback_retry,-)-from skyvern.forge.sdk.workflow.models.workflow import WorkflowRun, WorkflowRunStatus--MODULE = "skyvern.forge.sdk.workflow.code_mode_fallback"---def _workflow_run(- *,- status: WorkflowRunStatus = WorkflowRunStatus.failed,- run_with: str | None = "code",- fallback_attempt: int | None = None,- parent_workflow_run_id: str | None = None,- debug_session_id: str | None = None,- copilot_session_id: str | None = None,- retried_from_workflow_run_id: str | None = None,-) -> WorkflowRun:- now = datetime.now(timezone.utc)- return WorkflowRun(- workflow_run_id="wr_failed",- workflow_id="wf_test",- workflow_permanent_id="wpid_test",- organization_id="org_test",- status=status,- run_with=run_with,- failure_reason="element not found",- fallback_attempt=fallback_attempt,- parent_workflow_run_id=parent_workflow_run_id,- debug_session_id=debug_session_id,- copilot_session_id=copilot_session_id,- retried_from_workflow_run_id=retried_from_workflow_run_id,- trigger_type=WorkflowRunTriggerType.api,- created_at=now,- modified_at=now,- )---def _mock_app(*, flag_enabled: bool = True, existing_retry: str | None = None, block_scoped: bool = False) -> MagicMock:- mock_app = MagicMock()- provider = SimpleNamespace(is_feature_enabled_cached=AsyncMock(return_value=flag_enabled))- mock_app.EXPERIMENTATION_PROVIDER = provider- mock_app.AGENT_FUNCTION.is_block_scoped_workflow_run = AsyncMock(return_value=block_scoped)- mock_app.AGENT_FUNCTION.strip_proxy_session_extra_http_headers = MagicMock(return_value=None)- mock_app.DATABASE.debug.has_block_run_for_workflow_run = AsyncMock(return_value=False)- mock_app.DATABASE.workflow_runs.get_workflow_run_retried_by = AsyncMock(return_value=existing_retry)- mock_app.DATABASE.workflow_runs.get_workflow_run_parameters = AsyncMock(return_value=[])- mock_app.DATABASE.organizations.get_organization = AsyncMock(- return_value=SimpleNamespace(organization_id="org_test")- )- mock_app.DATABASE.tags.get_active_grouped_tags_for_run = AsyncMock(return_value={})- mock_app.WORKFLOW_SERVICE.get_workflow = AsyncMock(- return_value=SimpleNamespace(version=7, workflow_permanent_id="wpid_test")- )- return mock_app--- "status,run_with,expected",- [- (WorkflowRunStatus.failed, "code", True),- (WorkflowRunStatus.terminated, "code", True),- (WorkflowRunStatus.timed_out, "code", False),- (WorkflowRunStatus.completed, "code", False),- (WorkflowRunStatus.failed, "agent", False),- (WorkflowRunStatus.failed, None, False),- ],-)-def test_trigger_matches(status: WorkflowRunStatus, run_with: str | None, expected: bool) -> None:- assert _trigger_matches(_workflow_run(status=status, run_with=run_with)) is expected---async def test_spawns_agent_retry_for_failed_code_run() -> None:- workflow_run = _workflow_run()- request = MagicMock()- with (- patch(f"{MODULE}.app", _mock_app()),- patch(- "skyvern.services.workflow_service.workflow_request_body_from_existing_run",- MagicMock(return_value=request),- ),- patch(- "skyvern.services.workflow_service.run_workflow",- AsyncMock(return_value=SimpleNamespace(workflow_run_id="wr_retry")),- ) as run_workflow,- ):- result = await maybe_start_code_mode_fallback_retry(workflow_run, "org_test")-- assert result == "wr_retry"- # The retry must run as a pure agent and drop the code run's browser handles.- assert request.run_with == "agent"- assert request.browser_session_id is None- assert request.browser_profile_id is None- _, kwargs = run_workflow.call_args- assert kwargs["retried_from_workflow_run_id"] == "wr_failed"- assert kwargs["fallback_attempt"] == 1---async def test_no_retry_when_flag_disabled() -> None:- with (- patch(f"{MODULE}.app", _mock_app(flag_enabled=False)),- patch("skyvern.services.workflow_service.run_workflow", AsyncMock()) as run_workflow,- ):- result = await maybe_start_code_mode_fallback_retry(_workflow_run(), "org_test")-- assert result is None- run_workflow.assert_not_called()---async def test_returns_existing_retry_without_spawning() -> None:- with (- patch(f"{MODULE}.app", _mock_app(existing_retry="wr_prior")),- patch("skyvern.services.workflow_service.run_workflow", AsyncMock()) as run_workflow,- ):- result = await maybe_start_code_mode_fallback_retry(_workflow_run(), "org_test")-- assert result == "wr_prior"- run_workflow.assert_not_called()---async def test_no_retry_when_block_scoped() -> None:- with (- patch(f"{MODULE}.app", _mock_app(block_scoped=True)),- patch("skyvern.services.workflow_service.run_workflow", AsyncMock()) as run_workflow,- ):- result = await maybe_start_code_mode_fallback_retry(_workflow_run(), "org_test")-- assert result is None- run_workflow.assert_not_called()--- "kwargs",- [- {"run_with": "agent"},- {"run_with": None},- {"status": WorkflowRunStatus.timed_out},- {"fallback_attempt": 1},- {"retried_from_workflow_run_id": "wr_orig"},- {"parent_workflow_run_id": "wr_parent"},- {"debug_session_id": "dbg_1"},- {"copilot_session_id": "cop_1"},- ],-)-async def test_ineligible_runs_never_spawn(kwargs: dict) -> None:- with (- patch(f"{MODULE}.app", _mock_app()),- patch("skyvern.services.workflow_service.run_workflow", AsyncMock()) as run_workflow,- ):- result = await maybe_start_code_mode_fallback_retry(_workflow_run(**kwargs), "org_test")-- assert result is None- run_workflow.assert_not_called()diff --git a/tests/unit/test_credential_fallback.py b/tests/unit/test_credential_fallback.pyindex f3c640a56..8d19986b8 100644--- a/tests/unit/test_credential_fallback.py+++ b/tests/unit/test_credential_fallback.py@@ -846,7 +846,7 @@ async def test_mark_workflow_run_as_failed_does_not_schedule_before_cleanup() ->service = WorkflowService()workflow_run = _workflow_run(failure_category=[{"category": "AUTH_FAILURE"}])service._update_workflow_run_status = AsyncMock(return_value=workflow_run) # type: ignore[method-assign]- service._schedule_run_fallback_retries = MagicMock() # type: ignore[method-assign]+ service._schedule_credential_fallback_retry = MagicMock() # type: ignore[method-assign]result = await service.mark_workflow_run_as_failed(workflow_run_id="wr_failed",@@ -855,7 +855,7 @@ async def test_mark_workflow_run_as_failed_does_not_schedule_before_cleanup() ->)assert result == workflow_run- service._schedule_run_fallback_retries.assert_not_called()+ service._schedule_credential_fallback_retry.assert_not_called()@pytest.mark.asyncio@@ -866,7 +866,7 @@ async def test_mark_workflow_run_as_terminated_does_not_schedule_before_cleanup(failure_category=[{"category": "AUTH_FAILURE"}],)service._update_workflow_run_status = AsyncMock(return_value=workflow_run) # type: ignore[method-assign]- service._schedule_run_fallback_retries = MagicMock() # type: ignore[method-assign]+ service._schedule_credential_fallback_retry = MagicMock() # type: ignore[method-assign]result = await service.mark_workflow_run_as_terminated(workflow_run_id="wr_failed",@@ -875,7 +875,7 @@ async def test_mark_workflow_run_as_terminated_does_not_schedule_before_cleanup()assert result == workflow_run- service._schedule_run_fallback_retries.assert_not_called()+ service._schedule_credential_fallback_retry.assert_not_called()@pytest.mark.asyncio@@ -883,7 +883,7 @@ async def test_clean_up_workflow_schedules_credential_fallback_retry() -> None:service = WorkflowService()workflow_run = _workflow_run(failure_category=[{"category": "AUTH_FAILURE"}])workflow = _workflow([])- service._schedule_run_fallback_retries = MagicMock() # type: ignore[method-assign]+ service._schedule_credential_fallback_retry = MagicMock() # type: ignore[method-assign]browser_cleanup_result = SimpleNamespace(browser_state=None,tasks=[],@@ -909,7 +909,7 @@ async def test_clean_up_workflow_schedules_credential_fallback_retry() -> None:browser_cleanup_result=browser_cleanup_result, # type: ignore[arg-type])- service._schedule_run_fallback_retries.assert_called_once_with(workflow_run)+ service._schedule_credential_fallback_retry.assert_called_once_with(workflow_run)@pytest.mark.asyncio@@ -919,7 +919,7 @@ async def test_clean_up_workflow_schedules_retry_even_when_webhook_raises() -> Nservice = WorkflowService()workflow_run = _workflow_run(failure_category=[{"category": "AUTH_FAILURE"}])workflow = _workflow([])- service._schedule_run_fallback_retries = MagicMock() # type: ignore[method-assign]+ service._schedule_credential_fallback_retry = MagicMock() # type: ignore[method-assign]service.execute_workflow_webhook = AsyncMock(side_effect=RuntimeError("webhook prep failed")) # type: ignore[method-assign]browser_cleanup_result = SimpleNamespace(browser_state=None,@@ -947,7 +947,7 @@ async def test_clean_up_workflow_schedules_retry_even_when_webhook_raises() -> Nbrowser_cleanup_result=browser_cleanup_result, # type: ignore[arg-type])- service._schedule_run_fallback_retries.assert_called_once_with(workflow_run)+ service._schedule_credential_fallback_retry.assert_called_once_with(workflow_run)@pytest.mark.asyncio@@ -957,7 +957,7 @@ async def test_clean_up_workflow_schedules_retry_when_earlier_cleanup_step_raiseservice = WorkflowService()workflow_run = _workflow_run(failure_category=[{"category": "AUTH_FAILURE"}])workflow = _workflow([])- service._schedule_run_fallback_retries = MagicMock() # type: ignore[method-assign]+ service._schedule_credential_fallback_retry = MagicMock() # type: ignore[method-assign]browser_cleanup_result = SimpleNamespace(browser_state=None,tasks=[],@@ -984,7 +984,7 @@ async def test_clean_up_workflow_schedules_retry_when_earlier_cleanup_step_raisebrowser_cleanup_result=browser_cleanup_result, # type: ignore[arg-type])- service._schedule_run_fallback_retries.assert_called_once_with(workflow_run)+ service._schedule_credential_fallback_retry.assert_called_once_with(workflow_run)@pytest.mark.asyncio@@ -995,4 +995,4 @@ async def test_fallback_hook_exception_does_not_raise() -> None:"skyvern.forge.sdk.workflow.service.maybe_start_credential_fallback_retry",AsyncMock(side_effect=RuntimeError("boom")),):- await WorkflowService._start_run_fallback_retries_best_effort(workflow_run)+ await WorkflowService._start_credential_fallback_retry_best_effort(workflow_run)diff --git a/tests/unit/test_should_run_script.py b/tests/unit/test_should_run_script.pyindex 9d712547e..f42573a71 100644--- a/tests/unit/test_should_run_script.py+++ b/tests/unit/test_should_run_script.py@@ -45,11 +45,7 @@ def _make_workflow()-def _make_run(- run_with: str | None = "agent",- workflow_run_id: str = "wr_test",- retried_from_workflow_run_id: str | None = None,-) -> WorkflowRun:+def _make_run(run_with: str | None = "agent", workflow_run_id: str = "wr_test") -> WorkflowRun:return WorkflowRun(workflow_run_id=workflow_run_id,workflow_id="wf_test",@@ -57,7 +53,6 @@ def _make_run(organization_id="org_test",status=WorkflowRunStatus.running,run_with=run_with,- retried_from_workflow_run_id=retried_from_workflow_run_id,created_at=datetime.now(timezone.utc),modified_at=datetime.now(timezone.utc),)@@ -235,83 +230,6 @@ class TestCodeModeGate:assert len(agent_function.calls) == 1-class _UpgradingAgentFunction(AgentFunction):- def __init__(self, upgrade: bool, keep_code: bool = True) -> None:- self.upgrade = upgrade- self.keep_code = keep_code- self.upgrade_calls: list[tuple[str, str, str]] = []-- async def should_upgrade_to_code_mode(self, *, workflow: Workflow, workflow_run: WorkflowRun) -> bool:- self.upgrade_calls.append(- (workflow_run.workflow_run_id, workflow_run.organization_id, workflow.workflow_permanent_id)- )- return self.upgrade-- async def should_keep_code_mode_for_workflow_run(self, *, workflow: Workflow, workflow_run: WorkflowRun) -> bool:- return self.keep_code---class TestCodeModeUpgrade:- """A flag-gated rollout can upgrade an agent-intended run to code."""-- @pytest.mark.asyncio- async def test_upgrade_flips_agent_to_code(self, monkeypatch):- agent_function = _UpgradingAgentFunction(upgrade=True, keep_code=True)- monkeypatch.setattr(app, "AGENT_FUNCTION", agent_function)- from skyvern.forge.sdk.workflow.service import WorkflowService-- wf = _make_workflow(run_with="agent")- wr = _make_run(run_with="agent")- assert await WorkflowService().should_run_script(wf, wr) is True- assert agent_function.upgrade_calls == [("wr_test", "org_test", "wpid_test")]-- @pytest.mark.asyncio- async def test_upgrade_disabled_stays_agent(self, monkeypatch):- agent_function = _UpgradingAgentFunction(upgrade=False)- monkeypatch.setattr(app, "AGENT_FUNCTION", agent_function)- from skyvern.forge.sdk.workflow.service import WorkflowService-- wf = _make_workflow(run_with="agent")- wr = _make_run(run_with="agent")- assert await WorkflowService().should_run_script(wf, wr) is False-- @pytest.mark.asyncio- async def test_upgraded_run_still_passes_keep_code_gate(self, monkeypatch):- """Upgrade True but keep-code gate False → still agent (gate can veto)."""- agent_function = _UpgradingAgentFunction(upgrade=True, keep_code=False)- monkeypatch.setattr(app, "AGENT_FUNCTION", agent_function)- from skyvern.forge.sdk.workflow.service import WorkflowService-- wf = _make_workflow(run_with="agent")- wr = _make_run(run_with="agent")- assert await WorkflowService().should_run_script(wf, wr) is False-- @pytest.mark.asyncio- async def test_upgrade_not_consulted_when_already_code(self, monkeypatch):- agent_function = _UpgradingAgentFunction(upgrade=False)- monkeypatch.setattr(app, "AGENT_FUNCTION", agent_function)- from skyvern.forge.sdk.workflow.service import WorkflowService-- wf = _make_workflow(run_with="code")- wr = _make_run(run_with="code")- assert await WorkflowService().should_run_script(wf, wr) is True- assert agent_function.upgrade_calls == []-- @pytest.mark.asyncio- async def test_fallback_retry_never_re_upgraded(self, monkeypatch):- # A code->agent fallback retry (run_with=agent, retried_from set) must never be re-upgraded- # to code, or the rollout would re-run the very script that just failed and the fallback- # would be a no-op for exactly the rollout population.- agent_function = _UpgradingAgentFunction(upgrade=True, keep_code=True)- monkeypatch.setattr(app, "AGENT_FUNCTION", agent_function)- from skyvern.forge.sdk.workflow.service import WorkflowService-- wf = _make_workflow(run_with="agent")- wr = _make_run(run_with="agent", retried_from_workflow_run_id="wr_orig")- assert await WorkflowService().should_run_script(wf, wr) is False- assert agent_function.upgrade_calls == []--class TestScriptReviewerGate:"""The script reviewer should only fire when the script was actually executed.