Skip to content

Automate browser based workflows with AI

Replace the Workday static script (deterministic) + flag-gated code-mode rollout (#7603)

Cindy Li62d14e9fbf86
diff --git a/skyvern/forge/agent_functions.py b/skyvern/forge/agent_functions.py
index 17fa33e35..7297082c3 100644
--- a/skyvern/forge/agent_functions.py
+++ b/skyvern/forge/agent_functions.py
@@ -802,6 +802,14 @@ class AgentFunction:
) -> bool:
return True
+ async def should_upgrade_to_code_mode(
+ self,
+ *,
+ workflow: "Workflow",
+ workflow_run: "WorkflowRun",
+ ) -> bool:
+ return False
+
async def resolve_mcp_code_only_mode(
self,
organization_id: str | None,
diff --git a/skyvern/forge/sdk/workflow/code_mode_fallback.py b/skyvern/forge/sdk/workflow/code_mode_fallback.py
new file mode 100644
index 000000000..99272c8ac
--- /dev/null
+++ b/skyvern/forge/sdk/workflow/code_mode_fallback.py
@@ -0,0 +1,217 @@
+from __future__ import annotations
+
+import structlog
+from sqlalchemy.exc import IntegrityError
+
+from skyvern.exceptions import WorkflowNotFound
+from skyvern.forge import app
+from skyvern.forge.sdk.core import skyvern_context
+from skyvern.forge.sdk.core.skyvern_context import SkyvernContext
+from skyvern.forge.sdk.workflow.models.validators import is_reserved_tag_key
+from skyvern.forge.sdk.workflow.models.workflow import WorkflowRun, WorkflowRunStatus
+
+LOG = structlog.get_logger()
+
+CODE_MODE_FALLBACK_RETRY_FLAG = "CODE_MODE_FALLBACK_RETRY"
+CODE_RUN_WITH = "code"
+AGENT_RUN_WITH = "agent"
+# A code-mode run that failed/terminated retries once as a pure-agent run. timed_out is excluded:
+# re-running the full (slower) agent path on an already-long run is rarely what the caller wants.
+#
+# Double-apply risk (accepted, flag-gated): the retry is a fresh full-workflow run, so a run that
+# submitted an application and then failed on a later block would submit again. A run that actually
+# completed never reaches here (only failed/terminated retry), which covers the common case; the
+# submitted-then-errored edge is real, so the feature stays off by default (CODE_MODE_FALLBACK_RETRY)
+# and is enabled per-org only with explicit owner sign-off that the target tolerates a possible
+# duplicate submission.
+RETRYABLE_STATUSES = frozenset({WorkflowRunStatus.failed, WorkflowRunStatus.terminated})
+
+
+async def _retry_enabled_for_organization(organization_id: str, workflow_run: WorkflowRun) -> bool:
+ """Gate the auto-retry rollout. Fails closed: a retry run costs credits and re-fires webhooks.
+
+ distinct_id = organization_id (deliberately NOT run-level), so an org is fully in-or-out
+ rather than retrying on a random subset of its runs. Mirrors credential_fallback.
+ """
+ provider = getattr(app, "EXPERIMENTATION_PROVIDER", None)
+ if not provider:
+ return False
+ try:
+ return bool(
+ await provider.is_feature_enabled_cached(
+ CODE_MODE_FALLBACK_RETRY_FLAG,
+ organization_id,
+ properties={
+ "organization_id": organization_id,
+ "workflow_permanent_id": workflow_run.workflow_permanent_id,
+ },
+ )
+ )
+ except Exception:
+ LOG.warning(
+ "code_mode_fallback.flag_error",
+ workflow_run_id=workflow_run.workflow_run_id,
+ organization_id=organization_id,
+ exc_info=True,
+ )
+ return False
+
+
+def _trigger_matches(workflow_run: WorkflowRun) -> bool:
+ """A code-mode run in a retryable terminal state is eligible; anything else is not."""
+ if workflow_run.status not in RETRYABLE_STATUSES:
+ return False
+ return (workflow_run.run_with or AGENT_RUN_WITH) == CODE_RUN_WITH
+
+
+async def _reload_user_run_metadata(workflow_run: WorkflowRun, organization_id: str) -> dict[str, str] | None:
+ """Reload the failed run's user-writable tags so the retry keeps them. Reserved keys are dropped —
+ they are re-derived per run. Mirrors credential_fallback / the manual retry path."""
+ try:
+ grouped = await app.DATABASE.tags.get_active_grouped_tags_for_run(
+ workflow_run_id=workflow_run.workflow_run_id,
+ organization_id=organization_id,
+ )
+ except Exception:
+ LOG.warning(
+ "Failed to reload run metadata for code-mode fallback retry; continuing without it",
+ workflow_run_id=workflow_run.workflow_run_id,
+ exc_info=True,
+ )
+ return None
+ if not grouped:
+ return None
+ return {key: value for key, value in grouped.items() if not is_reserved_tag_key(key)} or None
+
+
+async def maybe_start_code_mode_fallback_retry(workflow_run: WorkflowRun, organization_id: str) -> str | None:
+ try:
+ return await _maybe_start_code_mode_fallback_retry(workflow_run, organization_id)
+ except Exception:
+ LOG.warning(
+ "Failed to start code-mode fallback retry",
+ workflow_run_id=workflow_run.workflow_run_id,
+ organization_id=organization_id,
+ exc_info=True,
+ )
+ return None
+
+
+async def _maybe_start_code_mode_fallback_retry(workflow_run: WorkflowRun, organization_id: str) -> str | None:
+ # Scheduled from clean_up_workflow, which runs for every terminal status. Bail on ineligible runs
+ # before the flag check and DB reads so the success path stays cheap.
+ if not _trigger_matches(workflow_run):
+ return None
+
+ # Never retry a retry: an agent attempt that fails is terminal, and a run that is already a
+ # fallback retry (credential or code) must not spawn another. This also bounds the chain.
+ if workflow_run.retried_from_workflow_run_id or workflow_run.fallback_attempt:
+ return None
+
+ # Block-run / copilot-test / debug runs supply block_labels directly and must not spawn a
+ # detached full-workflow run (real credits, real side effects) outside the test.
+ if workflow_run.parent_workflow_run_id or workflow_run.debug_session_id or workflow_run.copilot_session_id:
+ return None
+
+ if not await _retry_enabled_for_organization(organization_id, workflow_run):
+ return None
+
+ is_block_scoped_run = await app.AGENT_FUNCTION.is_block_scoped_workflow_run(workflow_run)
+ if not is_block_scoped_run:
+ is_block_scoped_run = await app.DATABASE.debug.has_block_run_for_workflow_run(
+ organization_id=organization_id,
+ workflow_run_id=workflow_run.workflow_run_id,
+ )
+ if is_block_scoped_run:
+ return None
+
+ # Idempotency: the retried_from_workflow_run_id unique index allows one retry per run, shared
+ # with credential_fallback — so if credential fallback already claimed the slot, no-op.
+ existing_retry_run_id = await app.DATABASE.workflow_runs.get_workflow_run_retried_by(
+ workflow_run_id=workflow_run.workflow_run_id,
+ organization_id=organization_id,
+ )
+ if existing_retry_run_id:
+ return existing_retry_run_id
+
+ try:
+ workflow = await app.WORKFLOW_SERVICE.get_workflow(
+ workflow_id=workflow_run.workflow_id,
+ organization_id=organization_id,
+ )
+ except WorkflowNotFound:
+ return None
+ if workflow is None:
+ return None
+
+ parameter_tuples = await app.DATABASE.workflow_runs.get_workflow_run_parameters(
+ workflow_run_id=workflow_run.workflow_run_id,
+ )
+ retry_parameters = {
+ workflow_parameter.key: workflow_run_parameter.value
+ for workflow_parameter, workflow_run_parameter in parameter_tuples
+ }
+
+ organization = await app.DATABASE.organizations.get_organization(organization_id)
+ if organization is None:
+ return None
+
+ from skyvern.services.workflow_service import run_workflow, workflow_request_body_from_existing_run
+
+ run_metadata = await _reload_user_run_metadata(workflow_run, organization_id)
+ workflow_request = workflow_request_body_from_existing_run(
+ workflow_run=workflow_run,
+ parameters=retry_parameters,
+ run_metadata=run_metadata,
+ )
+ # The whole point of the fallback: the retry runs as a pure agent, never code.
+ workflow_request.run_with = AGENT_RUN_WITH
+ # Shed every handle to the failed run's browser so the agent starts from a clean session instead
+ # of reconnecting to the code run's half-filled page / authenticated state.
+ workflow_request.browser_session_id = None
+ workflow_request.browser_profile_id = None
+ workflow_request.browser_address = None
+ workflow_request.cdp_connect_headers = None
+ workflow_request.extra_http_headers = app.AGENT_FUNCTION.strip_proxy_session_extra_http_headers(
+ workflow_request.extra_http_headers
+ )
+
+ next_attempt = (workflow_run.fallback_attempt or 0) + 1
+ try:
+ # A fresh context keeps run creation (and the background execution task it spawns, which
+ # inherits contextvars) from attributing the retry to the failed run.
+ with skyvern_context.scoped(SkyvernContext()):
+ retry_run = await run_workflow(
+ workflow_id=workflow_run.workflow_permanent_id,
+ organization=organization,
+ workflow_request=workflow_request,
+ template=False,
+ version=workflow.version,
+ trigger_type=workflow_run.trigger_type,
+ workflow_schedule_id=workflow_run.workflow_schedule_id,
+ retried_from_workflow_run_id=workflow_run.workflow_run_id,
+ fallback_attempt=next_attempt,
+ ignore_inherited_workflow_system_prompt=workflow_run.ignore_inherited_workflow_system_prompt,
+ )
+ except IntegrityError as exc:
+ error_text = str(exc.orig) if exc.orig is not None else str(exc)
+ if (
+ "ix_workflow_runs_retried_from_workflow_run_id" not in error_text
+ and "retried_from_workflow_run_id" not in error_text
+ ):
+ raise
+ existing_retry_run_id = await app.DATABASE.workflow_runs.get_workflow_run_retried_by(
+ workflow_run_id=workflow_run.workflow_run_id,
+ organization_id=organization_id,
+ )
+ if not existing_retry_run_id:
+ raise
+ return existing_retry_run_id
+
+ LOG.info(
+ "Started code-mode fallback retry",
+ workflow_run_id=workflow_run.workflow_run_id,
+ new_run_id=retry_run.workflow_run_id,
+ fallback_attempt=next_attempt,
+ )
+ return retry_run.workflow_run_id
diff --git a/skyvern/forge/sdk/workflow/service.py b/skyvern/forge/sdk/workflow/service.py
index d2eb87c1e..918f7ce83 100644
--- a/skyvern/forge/sdk/workflow/service.py
+++ b/skyvern/forge/sdk/workflow/service.py
@@ -83,6 +83,7 @@ from skyvern.forge.sdk.workflow.browser_profile_key import (
build_workflow_browser_session_storage_key,
render_browser_profile_key,
)
+from skyvern.forge.sdk.workflow.code_mode_fallback import maybe_start_code_mode_fallback_retry
from skyvern.forge.sdk.workflow.credential_fallback import (
VALID_FALLBACK_TRIGGERS,
maybe_start_credential_fallback_retry,
@@ -737,9 +738,12 @@ class WorkflowService:
)
@staticmethod
- async def _start_credential_fallback_retry_best_effort(workflow_run: WorkflowRun) -> None:
+ async def _start_run_fallback_retries_best_effort(workflow_run: WorkflowRun) -> None:
+ # Credential fallback first: it targets auth failures specifically. If it claims the
+ # single retry slot (retried_from unique index), code-mode fallback no-ops. Run them
+ # sequentially in one task so they can't race to create the retry.
try:
- await maybe_start_credential_fallback_retry(workflow_run, workflow_run.organization_id)
+ retry_run_id = await maybe_start_credential_fallback_retry(workflow_run, workflow_run.organization_id)
except Exception:
LOG.warning(
"Credential fallback retry hook failed",
@@ -747,9 +751,21 @@ class WorkflowService:
organization_id=workflow_run.organization_id,
exc_info=True,
)
+ retry_run_id = None
+ if retry_run_id:
+ return
+ try:
+ await maybe_start_code_mode_fallback_retry(workflow_run, workflow_run.organization_id)
+ except Exception:
+ LOG.warning(
+ "Code-mode fallback retry hook failed",
+ workflow_run_id=workflow_run.workflow_run_id,
+ organization_id=workflow_run.organization_id,
+ exc_info=True,
+ )
- def _schedule_credential_fallback_retry(self, workflow_run: WorkflowRun) -> None:
- task = asyncio.create_task(self._start_credential_fallback_retry_best_effort(workflow_run))
+ def _schedule_run_fallback_retries(self, workflow_run: WorkflowRun) -> None:
+ task = asyncio.create_task(self._start_run_fallback_retries_best_effort(workflow_run))
self._background_tasks.add(task)
task.add_done_callback(self._background_tasks.discard)
@@ -7465,13 +7481,14 @@ class WorkflowService:
for child_workflow_run_id in child_workflow_run_ids:
app.WORKFLOW_CONTEXT_MANAGER.remove_workflow_run_context(child_workflow_run_id)
- # Schedule the credential fallback retry only after this run's cleanup (artifact/video
- # persistence, browser/session write-back, webhook) has run, so the replacement run
- # cannot overlap it and race browser-profile/session writes. In the finally wrapping the
- # whole cleanup sequence — not after it — so an exception in any cleanup step can't skip
- # it; scheduling was removed from the status markers, which fired before cleanup and for
- # cascade/reaper failures that never reach cleanup. A no-op for non-eligible runs.
- self._schedule_credential_fallback_retry(workflow_run)
+ # Schedule the fallback retries (credential, then code→agent) only after this run's
+ # cleanup (artifact/video persistence, browser/session write-back, webhook) has run, so
+ # the replacement run cannot overlap it and race browser-profile/session writes. In the
+ # finally wrapping the whole cleanup sequence — not after it — so an exception in any
+ # cleanup step can't skip it; scheduling was removed from the status markers, which fired
+ # before cleanup and for cascade/reaper failures that never reach cleanup. A no-op for
+ # non-eligible runs.
+ self._schedule_run_fallback_retries(workflow_run)
async def prepare_workflow_webhook(
self,
@@ -9162,12 +9179,26 @@ class WorkflowService:
"""Whether this run should attempt cached-script execution.
Priority: run-level run_with > workflow-level run_with. Intended-code
- runs are then passed through the app-level code-mode gate.
+ runs are then passed through the app-level code-mode gate. Runs not
+ already intended as code may be upgraded by a flag-gated rollout;
+ an upgraded run has no matching script for a non-ATS workflow and so
+ degrades cleanly to agent per block.
"""
if workflow_run.run_with is not None:
intended_code = workflow_run.run_with == "code"
else:
intended_code = workflow.run_with == "code"
+ # A fallback retry has already made a deliberate execution-mode choice — in particular the
+ # code->agent fallback sets run_with=agent precisely so the retry does NOT run the script
+ # that just failed. The rollout must not re-upgrade it back to code, or the fallback would be
+ # a no-op for exactly the rollout population. Gate on the retry marker rather than
+ # `run_with is None`: normal runs inherit run_with="agent" from the workflow, so keying on
+ # None would disable the rollout for every ordinary run.
+ if not intended_code and workflow_run.retried_from_workflow_run_id is None:
+ intended_code = await app.AGENT_FUNCTION.should_upgrade_to_code_mode(
+ workflow=workflow,
+ workflow_run=workflow_run,
+ )
if not intended_code:
return False
return await app.AGENT_FUNCTION.should_keep_code_mode_for_workflow_run(
diff --git a/tests/unit/test_code_mode_fallback.py b/tests/unit/test_code_mode_fallback.py
new file mode 100644
index 000000000..c539ce668
--- /dev/null
+++ b/tests/unit/test_code_mode_fallback.py
@@ -0,0 +1,168 @@
+from __future__ import annotations
+
+from datetime import datetime, timezone
+from types import SimpleNamespace
+from unittest.mock import AsyncMock, MagicMock, patch
+
+import pytest
+
+from skyvern.forge.sdk.db.enums import WorkflowRunTriggerType
+from skyvern.forge.sdk.workflow.code_mode_fallback import (
+ _trigger_matches,
+ maybe_start_code_mode_fallback_retry,
+)
+from skyvern.forge.sdk.workflow.models.workflow import WorkflowRun, WorkflowRunStatus
+
+MODULE = "skyvern.forge.sdk.workflow.code_mode_fallback"
+
+
+def _workflow_run(
+ *,
+ status: WorkflowRunStatus = WorkflowRunStatus.failed,
+ run_with: str | None = "code",
+ fallback_attempt: int | None = None,
+ parent_workflow_run_id: str | None = None,
+ debug_session_id: str | None = None,
+ copilot_session_id: str | None = None,
+ retried_from_workflow_run_id: str | None = None,
+) -> WorkflowRun:
+ now = datetime.now(timezone.utc)
+ return WorkflowRun(
+ workflow_run_id="wr_failed",
+ workflow_id="wf_test",
+ workflow_permanent_id="wpid_test",
+ organization_id="org_test",
+ status=status,
+ run_with=run_with,
+ failure_reason="element not found",
+ fallback_attempt=fallback_attempt,
+ parent_workflow_run_id=parent_workflow_run_id,
+ debug_session_id=debug_session_id,
+ copilot_session_id=copilot_session_id,
+ retried_from_workflow_run_id=retried_from_workflow_run_id,
+ trigger_type=WorkflowRunTriggerType.api,
+ created_at=now,
+ modified_at=now,
+ )
+
+
+def _mock_app(*, flag_enabled: bool = True, existing_retry: str | None = None, block_scoped: bool = False) -> MagicMock:
+ mock_app = MagicMock()
+ provider = SimpleNamespace(is_feature_enabled_cached=AsyncMock(return_value=flag_enabled))
+ mock_app.EXPERIMENTATION_PROVIDER = provider
+ mock_app.AGENT_FUNCTION.is_block_scoped_workflow_run = AsyncMock(return_value=block_scoped)
+ mock_app.AGENT_FUNCTION.strip_proxy_session_extra_http_headers = MagicMock(return_value=None)
+ mock_app.DATABASE.debug.has_block_run_for_workflow_run = AsyncMock(return_value=False)
+ mock_app.DATABASE.workflow_runs.get_workflow_run_retried_by = AsyncMock(return_value=existing_retry)
+ mock_app.DATABASE.workflow_runs.get_workflow_run_parameters = AsyncMock(return_value=[])
+ mock_app.DATABASE.organizations.get_organization = AsyncMock(
+ return_value=SimpleNamespace(organization_id="org_test")
+ )
+ mock_app.DATABASE.tags.get_active_grouped_tags_for_run = AsyncMock(return_value={})
+ mock_app.WORKFLOW_SERVICE.get_workflow = AsyncMock(
+ return_value=SimpleNamespace(version=7, workflow_permanent_id="wpid_test")
+ )
+ return mock_app
+
+
+ "status,run_with,expected",
+ [
+ (WorkflowRunStatus.failed, "code", True),
+ (WorkflowRunStatus.terminated, "code", True),
+ (WorkflowRunStatus.timed_out, "code", False),
+ (WorkflowRunStatus.completed, "code", False),
+ (WorkflowRunStatus.failed, "agent", False),
+ (WorkflowRunStatus.failed, None, False),
+ ],
+)
+def test_trigger_matches(status: WorkflowRunStatus, run_with: str | None, expected: bool) -> None:
+ assert _trigger_matches(_workflow_run(status=status, run_with=run_with)) is expected
+
+
+async def test_spawns_agent_retry_for_failed_code_run() -> None:
+ workflow_run = _workflow_run()
+ request = MagicMock()
+ with (
+ patch(f"{MODULE}.app", _mock_app()),
+ patch(
+ "skyvern.services.workflow_service.workflow_request_body_from_existing_run",
+ MagicMock(return_value=request),
+ ),
+ patch(
+ "skyvern.services.workflow_service.run_workflow",
+ AsyncMock(return_value=SimpleNamespace(workflow_run_id="wr_retry")),
+ ) as run_workflow,
+ ):
+ result = await maybe_start_code_mode_fallback_retry(workflow_run, "org_test")
+
+ assert result == "wr_retry"
+ # The retry must run as a pure agent and drop the code run's browser handles.
+ assert request.run_with == "agent"
+ assert request.browser_session_id is None
+ assert request.browser_profile_id is None
+ _, kwargs = run_workflow.call_args
+ assert kwargs["retried_from_workflow_run_id"] == "wr_failed"
+ assert kwargs["fallback_attempt"] == 1
+
+
+async def test_no_retry_when_flag_disabled() -> None:
+ with (
+ patch(f"{MODULE}.app", _mock_app(flag_enabled=False)),
+ patch("skyvern.services.workflow_service.run_workflow", AsyncMock()) as run_workflow,
+ ):
+ result = await maybe_start_code_mode_fallback_retry(_workflow_run(), "org_test")
+
+ assert result is None
+ run_workflow.assert_not_called()
+
+
+async def test_returns_existing_retry_without_spawning() -> None:
+ with (
+ patch(f"{MODULE}.app", _mock_app(existing_retry="wr_prior")),
+ patch("skyvern.services.workflow_service.run_workflow", AsyncMock()) as run_workflow,
+ ):
+ result = await maybe_start_code_mode_fallback_retry(_workflow_run(), "org_test")
+
+ assert result == "wr_prior"
+ run_workflow.assert_not_called()
+
+
+async def test_no_retry_when_block_scoped() -> None:
+ with (
+ patch(f"{MODULE}.app", _mock_app(block_scoped=True)),
+ patch("skyvern.services.workflow_service.run_workflow", AsyncMock()) as run_workflow,
+ ):
+ result = await maybe_start_code_mode_fallback_retry(_workflow_run(), "org_test")
+
+ assert result is None
+ run_workflow.assert_not_called()
+
+
+ "kwargs",
+ [
+ {"run_with": "agent"},
+ {"run_with": None},
+ {"status": WorkflowRunStatus.timed_out},
+ {"fallback_attempt": 1},
+ {"retried_from_workflow_run_id": "wr_orig"},
+ {"parent_workflow_run_id": "wr_parent"},
+ {"debug_session_id": "dbg_1"},
+ {"copilot_session_id": "cop_1"},
+ ],
+)
+async def test_ineligible_runs_never_spawn(kwargs: dict) -> None:
+ with (
+ patch(f"{MODULE}.app", _mock_app()),
+ patch("skyvern.services.workflow_service.run_workflow", AsyncMock()) as run_workflow,
+ ):
+ result = await maybe_start_code_mode_fallback_retry(_workflow_run(**kwargs), "org_test")
+
+ assert result is None
+ run_workflow.assert_not_called()
diff --git a/tests/unit/test_credential_fallback.py b/tests/unit/test_credential_fallback.py
index 8d19986b8..f3c640a56 100644
--- a/tests/unit/test_credential_fallback.py
+++ b/tests/unit/test_credential_fallback.py
@@ -846,7 +846,7 @@ async def test_mark_workflow_run_as_failed_does_not_schedule_before_cleanup() ->
service = WorkflowService()
workflow_run = _workflow_run(failure_category=[{"category": "AUTH_FAILURE"}])
service._update_workflow_run_status = AsyncMock(return_value=workflow_run) # type: ignore[method-assign]
- service._schedule_credential_fallback_retry = MagicMock() # type: ignore[method-assign]
+ service._schedule_run_fallback_retries = MagicMock() # type: ignore[method-assign]
result = await service.mark_workflow_run_as_failed(
workflow_run_id="wr_failed",
@@ -855,7 +855,7 @@ async def test_mark_workflow_run_as_failed_does_not_schedule_before_cleanup() ->
)
assert result == workflow_run
- service._schedule_credential_fallback_retry.assert_not_called()
+ service._schedule_run_fallback_retries.assert_not_called()
@pytest.mark.asyncio
@@ -866,7 +866,7 @@ async def test_mark_workflow_run_as_terminated_does_not_schedule_before_cleanup(
failure_category=[{"category": "AUTH_FAILURE"}],
)
service._update_workflow_run_status = AsyncMock(return_value=workflow_run) # type: ignore[method-assign]
- service._schedule_credential_fallback_retry = MagicMock() # type: ignore[method-assign]
+ service._schedule_run_fallback_retries = MagicMock() # type: ignore[method-assign]
result = await service.mark_workflow_run_as_terminated(
workflow_run_id="wr_failed",
@@ -875,7 +875,7 @@ async def test_mark_workflow_run_as_terminated_does_not_schedule_before_cleanup(
)
assert result == workflow_run
- service._schedule_credential_fallback_retry.assert_not_called()
+ service._schedule_run_fallback_retries.assert_not_called()
@pytest.mark.asyncio
@@ -883,7 +883,7 @@ async def test_clean_up_workflow_schedules_credential_fallback_retry() -> None:
service = WorkflowService()
workflow_run = _workflow_run(failure_category=[{"category": "AUTH_FAILURE"}])
workflow = _workflow([])
- service._schedule_credential_fallback_retry = MagicMock() # type: ignore[method-assign]
+ service._schedule_run_fallback_retries = MagicMock() # type: ignore[method-assign]
browser_cleanup_result = SimpleNamespace(
browser_state=None,
tasks=[],
@@ -909,7 +909,7 @@ async def test_clean_up_workflow_schedules_credential_fallback_retry() -> None:
browser_cleanup_result=browser_cleanup_result, # type: ignore[arg-type]
)
- service._schedule_credential_fallback_retry.assert_called_once_with(workflow_run)
+ service._schedule_run_fallback_retries.assert_called_once_with(workflow_run)
@pytest.mark.asyncio
@@ -919,7 +919,7 @@ async def test_clean_up_workflow_schedules_retry_even_when_webhook_raises() -> N
service = WorkflowService()
workflow_run = _workflow_run(failure_category=[{"category": "AUTH_FAILURE"}])
workflow = _workflow([])
- service._schedule_credential_fallback_retry = MagicMock() # type: ignore[method-assign]
+ service._schedule_run_fallback_retries = MagicMock() # type: ignore[method-assign]
service.execute_workflow_webhook = AsyncMock(side_effect=RuntimeError("webhook prep failed")) # type: ignore[method-assign]
browser_cleanup_result = SimpleNamespace(
browser_state=None,
@@ -947,7 +947,7 @@ async def test_clean_up_workflow_schedules_retry_even_when_webhook_raises() -> N
browser_cleanup_result=browser_cleanup_result, # type: ignore[arg-type]
)
- service._schedule_credential_fallback_retry.assert_called_once_with(workflow_run)
+ service._schedule_run_fallback_retries.assert_called_once_with(workflow_run)
@pytest.mark.asyncio
@@ -957,7 +957,7 @@ async def test_clean_up_workflow_schedules_retry_when_earlier_cleanup_step_raise
service = WorkflowService()
workflow_run = _workflow_run(failure_category=[{"category": "AUTH_FAILURE"}])
workflow = _workflow([])
- service._schedule_credential_fallback_retry = MagicMock() # type: ignore[method-assign]
+ service._schedule_run_fallback_retries = MagicMock() # type: ignore[method-assign]
browser_cleanup_result = SimpleNamespace(
browser_state=None,
tasks=[],
@@ -984,7 +984,7 @@ async def test_clean_up_workflow_schedules_retry_when_earlier_cleanup_step_raise
browser_cleanup_result=browser_cleanup_result, # type: ignore[arg-type]
)
- service._schedule_credential_fallback_retry.assert_called_once_with(workflow_run)
+ service._schedule_run_fallback_retries.assert_called_once_with(workflow_run)
@pytest.mark.asyncio
@@ -995,4 +995,4 @@ async def test_fallback_hook_exception_does_not_raise() -> None:
"skyvern.forge.sdk.workflow.service.maybe_start_credential_fallback_retry",
AsyncMock(side_effect=RuntimeError("boom")),
):
- await WorkflowService._start_credential_fallback_retry_best_effort(workflow_run)
+ await WorkflowService._start_run_fallback_retries_best_effort(workflow_run)
diff --git a/tests/unit/test_should_run_script.py b/tests/unit/test_should_run_script.py
index f42573a71..9d712547e 100644
--- a/tests/unit/test_should_run_script.py
+++ b/tests/unit/test_should_run_script.py
@@ -45,7 +45,11 @@ def _make_workflow(
)
-def _make_run(run_with: str | None = "agent", workflow_run_id: str = "wr_test") -> WorkflowRun:
+def _make_run(
+ run_with: str | None = "agent",
+ workflow_run_id: str = "wr_test",
+ retried_from_workflow_run_id: str | None = None,
+) -> WorkflowRun:
return WorkflowRun(
workflow_run_id=workflow_run_id,
workflow_id="wf_test",
@@ -53,6 +57,7 @@ def _make_run(run_with: str | None = "agent", workflow_run_id: str = "wr_test")
organization_id="org_test",
status=WorkflowRunStatus.running,
run_with=run_with,
+ retried_from_workflow_run_id=retried_from_workflow_run_id,
created_at=datetime.now(timezone.utc),
modified_at=datetime.now(timezone.utc),
)
@@ -230,6 +235,83 @@ class TestCodeModeGate:
assert len(agent_function.calls) == 1
+class _UpgradingAgentFunction(AgentFunction):
+ def __init__(self, upgrade: bool, keep_code: bool = True) -> None:
+ self.upgrade = upgrade
+ self.keep_code = keep_code
+ self.upgrade_calls: list[tuple[str, str, str]] = []
+
+ async def should_upgrade_to_code_mode(self, *, workflow: Workflow, workflow_run: WorkflowRun) -> bool:
+ self.upgrade_calls.append(
+ (workflow_run.workflow_run_id, workflow_run.organization_id, workflow.workflow_permanent_id)
+ )
+ return self.upgrade
+
+ async def should_keep_code_mode_for_workflow_run(self, *, workflow: Workflow, workflow_run: WorkflowRun) -> bool:
+ return self.keep_code
+
+
+class TestCodeModeUpgrade:
+ """A flag-gated rollout can upgrade an agent-intended run to code."""
+
+ @pytest.mark.asyncio
+ async def test_upgrade_flips_agent_to_code(self, monkeypatch):
+ agent_function = _UpgradingAgentFunction(upgrade=True, keep_code=True)
+ monkeypatch.setattr(app, "AGENT_FUNCTION", agent_function)
+ from skyvern.forge.sdk.workflow.service import WorkflowService
+
+ wf = _make_workflow(run_with="agent")
+ wr = _make_run(run_with="agent")
+ assert await WorkflowService().should_run_script(wf, wr) is True
+ assert agent_function.upgrade_calls == [("wr_test", "org_test", "wpid_test")]
+
+ @pytest.mark.asyncio
+ async def test_upgrade_disabled_stays_agent(self, monkeypatch):
+ agent_function = _UpgradingAgentFunction(upgrade=False)
+ monkeypatch.setattr(app, "AGENT_FUNCTION", agent_function)
+ from skyvern.forge.sdk.workflow.service import WorkflowService
+
+ wf = _make_workflow(run_with="agent")
+ wr = _make_run(run_with="agent")
+ assert await WorkflowService().should_run_script(wf, wr) is False
+
+ @pytest.mark.asyncio
+ async def test_upgraded_run_still_passes_keep_code_gate(self, monkeypatch):
+ """Upgrade True but keep-code gate False → still agent (gate can veto)."""
+ agent_function = _UpgradingAgentFunction(upgrade=True, keep_code=False)
+ monkeypatch.setattr(app, "AGENT_FUNCTION", agent_function)
+ from skyvern.forge.sdk.workflow.service import WorkflowService
+
+ wf = _make_workflow(run_with="agent")
+ wr = _make_run(run_with="agent")
+ assert await WorkflowService().should_run_script(wf, wr) is False
+
+ @pytest.mark.asyncio
+ async def test_upgrade_not_consulted_when_already_code(self, monkeypatch):
+ agent_function = _UpgradingAgentFunction(upgrade=False)
+ monkeypatch.setattr(app, "AGENT_FUNCTION", agent_function)
+ from skyvern.forge.sdk.workflow.service import WorkflowService
+
+ wf = _make_workflow(run_with="code")
+ wr = _make_run(run_with="code")
+ assert await WorkflowService().should_run_script(wf, wr) is True
+ assert agent_function.upgrade_calls == []
+
+ @pytest.mark.asyncio
+ async def test_fallback_retry_never_re_upgraded(self, monkeypatch):
+ # A code->agent fallback retry (run_with=agent, retried_from set) must never be re-upgraded
+ # to code, or the rollout would re-run the very script that just failed and the fallback
+ # would be a no-op for exactly the rollout population.
+ agent_function = _UpgradingAgentFunction(upgrade=True, keep_code=True)
+ monkeypatch.setattr(app, "AGENT_FUNCTION", agent_function)
+ from skyvern.forge.sdk.workflow.service import WorkflowService
+
+ wf = _make_workflow(run_with="agent")
+ wr = _make_run(run_with="agent", retried_from_workflow_run_id="wr_orig")
+ assert await WorkflowService().should_run_script(wf, wr) is False
+ assert agent_function.upgrade_calls == []
+
+
class TestScriptReviewerGate:
"""The script reviewer should only fire when the script was actually executed.