Skip to content

Automate browser based workflows with AI

Revert "Replace the Workday static script (deterministic) + flag-gated code-mode rollout" (#7620)

Cindy Li2f29ec12dd8b
diff --git a/skyvern/forge/agent_functions.py b/skyvern/forge/agent_functions.py
index 7297082c3..17fa33e35 100644
--- a/skyvern/forge/agent_functions.py
+++ b/skyvern/forge/agent_functions.py
@@ -802,14 +802,6 @@ class AgentFunction:
) -> bool:
return True
- async def should_upgrade_to_code_mode(
- self,
- *,
- workflow: "Workflow",
- workflow_run: "WorkflowRun",
- ) -> bool:
- return False
-
async def resolve_mcp_code_only_mode(
self,
organization_id: str | None,
diff --git a/skyvern/forge/sdk/workflow/code_mode_fallback.py b/skyvern/forge/sdk/workflow/code_mode_fallback.py
deleted file mode 100644
index 99272c8ac..000000000
--- a/skyvern/forge/sdk/workflow/code_mode_fallback.py
+++ /dev/null
@@ -1,217 +0,0 @@
-from __future__ import annotations
-
-import structlog
-from sqlalchemy.exc import IntegrityError
-
-from skyvern.exceptions import WorkflowNotFound
-from skyvern.forge import app
-from skyvern.forge.sdk.core import skyvern_context
-from skyvern.forge.sdk.core.skyvern_context import SkyvernContext
-from skyvern.forge.sdk.workflow.models.validators import is_reserved_tag_key
-from skyvern.forge.sdk.workflow.models.workflow import WorkflowRun, WorkflowRunStatus
-
-LOG = structlog.get_logger()
-
-CODE_MODE_FALLBACK_RETRY_FLAG = "CODE_MODE_FALLBACK_RETRY"
-CODE_RUN_WITH = "code"
-AGENT_RUN_WITH = "agent"
-# A code-mode run that failed/terminated retries once as a pure-agent run. timed_out is excluded:
-# re-running the full (slower) agent path on an already-long run is rarely what the caller wants.
-#
-# Double-apply risk (accepted, flag-gated): the retry is a fresh full-workflow run, so a run that
-# submitted an application and then failed on a later block would submit again. A run that actually
-# completed never reaches here (only failed/terminated retry), which covers the common case; the
-# submitted-then-errored edge is real, so the feature stays off by default (CODE_MODE_FALLBACK_RETRY)
-# and is enabled per-org only with explicit owner sign-off that the target tolerates a possible
-# duplicate submission.
-RETRYABLE_STATUSES = frozenset({WorkflowRunStatus.failed, WorkflowRunStatus.terminated})
-
-
-async def _retry_enabled_for_organization(organization_id: str, workflow_run: WorkflowRun) -> bool:
- """Gate the auto-retry rollout. Fails closed: a retry run costs credits and re-fires webhooks.
-
- distinct_id = organization_id (deliberately NOT run-level), so an org is fully in-or-out
- rather than retrying on a random subset of its runs. Mirrors credential_fallback.
- """
- provider = getattr(app, "EXPERIMENTATION_PROVIDER", None)
- if not provider:
- return False
- try:
- return bool(
- await provider.is_feature_enabled_cached(
- CODE_MODE_FALLBACK_RETRY_FLAG,
- organization_id,
- properties={
- "organization_id": organization_id,
- "workflow_permanent_id": workflow_run.workflow_permanent_id,
- },
- )
- )
- except Exception:
- LOG.warning(
- "code_mode_fallback.flag_error",
- workflow_run_id=workflow_run.workflow_run_id,
- organization_id=organization_id,
- exc_info=True,
- )
- return False
-
-
-def _trigger_matches(workflow_run: WorkflowRun) -> bool:
- """A code-mode run in a retryable terminal state is eligible; anything else is not."""
- if workflow_run.status not in RETRYABLE_STATUSES:
- return False
- return (workflow_run.run_with or AGENT_RUN_WITH) == CODE_RUN_WITH
-
-
-async def _reload_user_run_metadata(workflow_run: WorkflowRun, organization_id: str) -> dict[str, str] | None:
- """Reload the failed run's user-writable tags so the retry keeps them. Reserved keys are dropped —
- they are re-derived per run. Mirrors credential_fallback / the manual retry path."""
- try:
- grouped = await app.DATABASE.tags.get_active_grouped_tags_for_run(
- workflow_run_id=workflow_run.workflow_run_id,
- organization_id=organization_id,
- )
- except Exception:
- LOG.warning(
- "Failed to reload run metadata for code-mode fallback retry; continuing without it",
- workflow_run_id=workflow_run.workflow_run_id,
- exc_info=True,
- )
- return None
- if not grouped:
- return None
- return {key: value for key, value in grouped.items() if not is_reserved_tag_key(key)} or None
-
-
-async def maybe_start_code_mode_fallback_retry(workflow_run: WorkflowRun, organization_id: str) -> str | None:
- try:
- return await _maybe_start_code_mode_fallback_retry(workflow_run, organization_id)
- except Exception:
- LOG.warning(
- "Failed to start code-mode fallback retry",
- workflow_run_id=workflow_run.workflow_run_id,
- organization_id=organization_id,
- exc_info=True,
- )
- return None
-
-
-async def _maybe_start_code_mode_fallback_retry(workflow_run: WorkflowRun, organization_id: str) -> str | None:
- # Scheduled from clean_up_workflow, which runs for every terminal status. Bail on ineligible runs
- # before the flag check and DB reads so the success path stays cheap.
- if not _trigger_matches(workflow_run):
- return None
-
- # Never retry a retry: an agent attempt that fails is terminal, and a run that is already a
- # fallback retry (credential or code) must not spawn another. This also bounds the chain.
- if workflow_run.retried_from_workflow_run_id or workflow_run.fallback_attempt:
- return None
-
- # Block-run / copilot-test / debug runs supply block_labels directly and must not spawn a
- # detached full-workflow run (real credits, real side effects) outside the test.
- if workflow_run.parent_workflow_run_id or workflow_run.debug_session_id or workflow_run.copilot_session_id:
- return None
-
- if not await _retry_enabled_for_organization(organization_id, workflow_run):
- return None
-
- is_block_scoped_run = await app.AGENT_FUNCTION.is_block_scoped_workflow_run(workflow_run)
- if not is_block_scoped_run:
- is_block_scoped_run = await app.DATABASE.debug.has_block_run_for_workflow_run(
- organization_id=organization_id,
- workflow_run_id=workflow_run.workflow_run_id,
- )
- if is_block_scoped_run:
- return None
-
- # Idempotency: the retried_from_workflow_run_id unique index allows one retry per run, shared
- # with credential_fallback — so if credential fallback already claimed the slot, no-op.
- existing_retry_run_id = await app.DATABASE.workflow_runs.get_workflow_run_retried_by(
- workflow_run_id=workflow_run.workflow_run_id,
- organization_id=organization_id,
- )
- if existing_retry_run_id:
- return existing_retry_run_id
-
- try:
- workflow = await app.WORKFLOW_SERVICE.get_workflow(
- workflow_id=workflow_run.workflow_id,
- organization_id=organization_id,
- )
- except WorkflowNotFound:
- return None
- if workflow is None:
- return None
-
- parameter_tuples = await app.DATABASE.workflow_runs.get_workflow_run_parameters(
- workflow_run_id=workflow_run.workflow_run_id,
- )
- retry_parameters = {
- workflow_parameter.key: workflow_run_parameter.value
- for workflow_parameter, workflow_run_parameter in parameter_tuples
- }
-
- organization = await app.DATABASE.organizations.get_organization(organization_id)
- if organization is None:
- return None
-
- from skyvern.services.workflow_service import run_workflow, workflow_request_body_from_existing_run
-
- run_metadata = await _reload_user_run_metadata(workflow_run, organization_id)
- workflow_request = workflow_request_body_from_existing_run(
- workflow_run=workflow_run,
- parameters=retry_parameters,
- run_metadata=run_metadata,
- )
- # The whole point of the fallback: the retry runs as a pure agent, never code.
- workflow_request.run_with = AGENT_RUN_WITH
- # Shed every handle to the failed run's browser so the agent starts from a clean session instead
- # of reconnecting to the code run's half-filled page / authenticated state.
- workflow_request.browser_session_id = None
- workflow_request.browser_profile_id = None
- workflow_request.browser_address = None
- workflow_request.cdp_connect_headers = None
- workflow_request.extra_http_headers = app.AGENT_FUNCTION.strip_proxy_session_extra_http_headers(
- workflow_request.extra_http_headers
- )
-
- next_attempt = (workflow_run.fallback_attempt or 0) + 1
- try:
- # A fresh context keeps run creation (and the background execution task it spawns, which
- # inherits contextvars) from attributing the retry to the failed run.
- with skyvern_context.scoped(SkyvernContext()):
- retry_run = await run_workflow(
- workflow_id=workflow_run.workflow_permanent_id,
- organization=organization,
- workflow_request=workflow_request,
- template=False,
- version=workflow.version,
- trigger_type=workflow_run.trigger_type,
- workflow_schedule_id=workflow_run.workflow_schedule_id,
- retried_from_workflow_run_id=workflow_run.workflow_run_id,
- fallback_attempt=next_attempt,
- ignore_inherited_workflow_system_prompt=workflow_run.ignore_inherited_workflow_system_prompt,
- )
- except IntegrityError as exc:
- error_text = str(exc.orig) if exc.orig is not None else str(exc)
- if (
- "ix_workflow_runs_retried_from_workflow_run_id" not in error_text
- and "retried_from_workflow_run_id" not in error_text
- ):
- raise
- existing_retry_run_id = await app.DATABASE.workflow_runs.get_workflow_run_retried_by(
- workflow_run_id=workflow_run.workflow_run_id,
- organization_id=organization_id,
- )
- if not existing_retry_run_id:
- raise
- return existing_retry_run_id
-
- LOG.info(
- "Started code-mode fallback retry",
- workflow_run_id=workflow_run.workflow_run_id,
- new_run_id=retry_run.workflow_run_id,
- fallback_attempt=next_attempt,
- )
- return retry_run.workflow_run_id
diff --git a/skyvern/forge/sdk/workflow/service.py b/skyvern/forge/sdk/workflow/service.py
index 918f7ce83..d2eb87c1e 100644
--- a/skyvern/forge/sdk/workflow/service.py
+++ b/skyvern/forge/sdk/workflow/service.py
@@ -83,7 +83,6 @@ from skyvern.forge.sdk.workflow.browser_profile_key import (
build_workflow_browser_session_storage_key,
render_browser_profile_key,
)
-from skyvern.forge.sdk.workflow.code_mode_fallback import maybe_start_code_mode_fallback_retry
from skyvern.forge.sdk.workflow.credential_fallback import (
VALID_FALLBACK_TRIGGERS,
maybe_start_credential_fallback_retry,
@@ -738,12 +737,9 @@ class WorkflowService:
)
@staticmethod
- async def _start_run_fallback_retries_best_effort(workflow_run: WorkflowRun) -> None:
- # Credential fallback first: it targets auth failures specifically. If it claims the
- # single retry slot (retried_from unique index), code-mode fallback no-ops. Run them
- # sequentially in one task so they can't race to create the retry.
+ async def _start_credential_fallback_retry_best_effort(workflow_run: WorkflowRun) -> None:
try:
- retry_run_id = await maybe_start_credential_fallback_retry(workflow_run, workflow_run.organization_id)
+ await maybe_start_credential_fallback_retry(workflow_run, workflow_run.organization_id)
except Exception:
LOG.warning(
"Credential fallback retry hook failed",
@@ -751,21 +747,9 @@ class WorkflowService:
organization_id=workflow_run.organization_id,
exc_info=True,
)
- retry_run_id = None
- if retry_run_id:
- return
- try:
- await maybe_start_code_mode_fallback_retry(workflow_run, workflow_run.organization_id)
- except Exception:
- LOG.warning(
- "Code-mode fallback retry hook failed",
- workflow_run_id=workflow_run.workflow_run_id,
- organization_id=workflow_run.organization_id,
- exc_info=True,
- )
- def _schedule_run_fallback_retries(self, workflow_run: WorkflowRun) -> None:
- task = asyncio.create_task(self._start_run_fallback_retries_best_effort(workflow_run))
+ def _schedule_credential_fallback_retry(self, workflow_run: WorkflowRun) -> None:
+ task = asyncio.create_task(self._start_credential_fallback_retry_best_effort(workflow_run))
self._background_tasks.add(task)
task.add_done_callback(self._background_tasks.discard)
@@ -7481,14 +7465,13 @@ class WorkflowService:
for child_workflow_run_id in child_workflow_run_ids:
app.WORKFLOW_CONTEXT_MANAGER.remove_workflow_run_context(child_workflow_run_id)
- # Schedule the fallback retries (credential, then code→agent) only after this run's
- # cleanup (artifact/video persistence, browser/session write-back, webhook) has run, so
- # the replacement run cannot overlap it and race browser-profile/session writes. In the
- # finally wrapping the whole cleanup sequence — not after it — so an exception in any
- # cleanup step can't skip it; scheduling was removed from the status markers, which fired
- # before cleanup and for cascade/reaper failures that never reach cleanup. A no-op for
- # non-eligible runs.
- self._schedule_run_fallback_retries(workflow_run)
+ # Schedule the credential fallback retry only after this run's cleanup (artifact/video
+ # persistence, browser/session write-back, webhook) has run, so the replacement run
+ # cannot overlap it and race browser-profile/session writes. In the finally wrapping the
+ # whole cleanup sequence — not after it — so an exception in any cleanup step can't skip
+ # it; scheduling was removed from the status markers, which fired before cleanup and for
+ # cascade/reaper failures that never reach cleanup. A no-op for non-eligible runs.
+ self._schedule_credential_fallback_retry(workflow_run)
async def prepare_workflow_webhook(
self,
@@ -9179,26 +9162,12 @@ class WorkflowService:
"""Whether this run should attempt cached-script execution.
Priority: run-level run_with > workflow-level run_with. Intended-code
- runs are then passed through the app-level code-mode gate. Runs not
- already intended as code may be upgraded by a flag-gated rollout;
- an upgraded run has no matching script for a non-ATS workflow and so
- degrades cleanly to agent per block.
+ runs are then passed through the app-level code-mode gate.
"""
if workflow_run.run_with is not None:
intended_code = workflow_run.run_with == "code"
else:
intended_code = workflow.run_with == "code"
- # A fallback retry has already made a deliberate execution-mode choice — in particular the
- # code->agent fallback sets run_with=agent precisely so the retry does NOT run the script
- # that just failed. The rollout must not re-upgrade it back to code, or the fallback would be
- # a no-op for exactly the rollout population. Gate on the retry marker rather than
- # `run_with is None`: normal runs inherit run_with="agent" from the workflow, so keying on
- # None would disable the rollout for every ordinary run.
- if not intended_code and workflow_run.retried_from_workflow_run_id is None:
- intended_code = await app.AGENT_FUNCTION.should_upgrade_to_code_mode(
- workflow=workflow,
- workflow_run=workflow_run,
- )
if not intended_code:
return False
return await app.AGENT_FUNCTION.should_keep_code_mode_for_workflow_run(
diff --git a/tests/unit/test_code_mode_fallback.py b/tests/unit/test_code_mode_fallback.py
deleted file mode 100644
index c539ce668..000000000
--- a/tests/unit/test_code_mode_fallback.py
+++ /dev/null
@@ -1,168 +0,0 @@
-from __future__ import annotations
-
-from datetime import datetime, timezone
-from types import SimpleNamespace
-from unittest.mock import AsyncMock, MagicMock, patch
-
-import pytest
-
-from skyvern.forge.sdk.db.enums import WorkflowRunTriggerType
-from skyvern.forge.sdk.workflow.code_mode_fallback import (
- _trigger_matches,
- maybe_start_code_mode_fallback_retry,
-)
-from skyvern.forge.sdk.workflow.models.workflow import WorkflowRun, WorkflowRunStatus
-
-MODULE = "skyvern.forge.sdk.workflow.code_mode_fallback"
-
-
-def _workflow_run(
- *,
- status: WorkflowRunStatus = WorkflowRunStatus.failed,
- run_with: str | None = "code",
- fallback_attempt: int | None = None,
- parent_workflow_run_id: str | None = None,
- debug_session_id: str | None = None,
- copilot_session_id: str | None = None,
- retried_from_workflow_run_id: str | None = None,
-) -> WorkflowRun:
- now = datetime.now(timezone.utc)
- return WorkflowRun(
- workflow_run_id="wr_failed",
- workflow_id="wf_test",
- workflow_permanent_id="wpid_test",
- organization_id="org_test",
- status=status,
- run_with=run_with,
- failure_reason="element not found",
- fallback_attempt=fallback_attempt,
- parent_workflow_run_id=parent_workflow_run_id,
- debug_session_id=debug_session_id,
- copilot_session_id=copilot_session_id,
- retried_from_workflow_run_id=retried_from_workflow_run_id,
- trigger_type=WorkflowRunTriggerType.api,
- created_at=now,
- modified_at=now,
- )
-
-
-def _mock_app(*, flag_enabled: bool = True, existing_retry: str | None = None, block_scoped: bool = False) -> MagicMock:
- mock_app = MagicMock()
- provider = SimpleNamespace(is_feature_enabled_cached=AsyncMock(return_value=flag_enabled))
- mock_app.EXPERIMENTATION_PROVIDER = provider
- mock_app.AGENT_FUNCTION.is_block_scoped_workflow_run = AsyncMock(return_value=block_scoped)
- mock_app.AGENT_FUNCTION.strip_proxy_session_extra_http_headers = MagicMock(return_value=None)
- mock_app.DATABASE.debug.has_block_run_for_workflow_run = AsyncMock(return_value=False)
- mock_app.DATABASE.workflow_runs.get_workflow_run_retried_by = AsyncMock(return_value=existing_retry)
- mock_app.DATABASE.workflow_runs.get_workflow_run_parameters = AsyncMock(return_value=[])
- mock_app.DATABASE.organizations.get_organization = AsyncMock(
- return_value=SimpleNamespace(organization_id="org_test")
- )
- mock_app.DATABASE.tags.get_active_grouped_tags_for_run = AsyncMock(return_value={})
- mock_app.WORKFLOW_SERVICE.get_workflow = AsyncMock(
- return_value=SimpleNamespace(version=7, workflow_permanent_id="wpid_test")
- )
- return mock_app
-
-
- "status,run_with,expected",
- [
- (WorkflowRunStatus.failed, "code", True),
- (WorkflowRunStatus.terminated, "code", True),
- (WorkflowRunStatus.timed_out, "code", False),
- (WorkflowRunStatus.completed, "code", False),
- (WorkflowRunStatus.failed, "agent", False),
- (WorkflowRunStatus.failed, None, False),
- ],
-)
-def test_trigger_matches(status: WorkflowRunStatus, run_with: str | None, expected: bool) -> None:
- assert _trigger_matches(_workflow_run(status=status, run_with=run_with)) is expected
-
-
-async def test_spawns_agent_retry_for_failed_code_run() -> None:
- workflow_run = _workflow_run()
- request = MagicMock()
- with (
- patch(f"{MODULE}.app", _mock_app()),
- patch(
- "skyvern.services.workflow_service.workflow_request_body_from_existing_run",
- MagicMock(return_value=request),
- ),
- patch(
- "skyvern.services.workflow_service.run_workflow",
- AsyncMock(return_value=SimpleNamespace(workflow_run_id="wr_retry")),
- ) as run_workflow,
- ):
- result = await maybe_start_code_mode_fallback_retry(workflow_run, "org_test")
-
- assert result == "wr_retry"
- # The retry must run as a pure agent and drop the code run's browser handles.
- assert request.run_with == "agent"
- assert request.browser_session_id is None
- assert request.browser_profile_id is None
- _, kwargs = run_workflow.call_args
- assert kwargs["retried_from_workflow_run_id"] == "wr_failed"
- assert kwargs["fallback_attempt"] == 1
-
-
-async def test_no_retry_when_flag_disabled() -> None:
- with (
- patch(f"{MODULE}.app", _mock_app(flag_enabled=False)),
- patch("skyvern.services.workflow_service.run_workflow", AsyncMock()) as run_workflow,
- ):
- result = await maybe_start_code_mode_fallback_retry(_workflow_run(), "org_test")
-
- assert result is None
- run_workflow.assert_not_called()
-
-
-async def test_returns_existing_retry_without_spawning() -> None:
- with (
- patch(f"{MODULE}.app", _mock_app(existing_retry="wr_prior")),
- patch("skyvern.services.workflow_service.run_workflow", AsyncMock()) as run_workflow,
- ):
- result = await maybe_start_code_mode_fallback_retry(_workflow_run(), "org_test")
-
- assert result == "wr_prior"
- run_workflow.assert_not_called()
-
-
-async def test_no_retry_when_block_scoped() -> None:
- with (
- patch(f"{MODULE}.app", _mock_app(block_scoped=True)),
- patch("skyvern.services.workflow_service.run_workflow", AsyncMock()) as run_workflow,
- ):
- result = await maybe_start_code_mode_fallback_retry(_workflow_run(), "org_test")
-
- assert result is None
- run_workflow.assert_not_called()
-
-
- "kwargs",
- [
- {"run_with": "agent"},
- {"run_with": None},
- {"status": WorkflowRunStatus.timed_out},
- {"fallback_attempt": 1},
- {"retried_from_workflow_run_id": "wr_orig"},
- {"parent_workflow_run_id": "wr_parent"},
- {"debug_session_id": "dbg_1"},
- {"copilot_session_id": "cop_1"},
- ],
-)
-async def test_ineligible_runs_never_spawn(kwargs: dict) -> None:
- with (
- patch(f"{MODULE}.app", _mock_app()),
- patch("skyvern.services.workflow_service.run_workflow", AsyncMock()) as run_workflow,
- ):
- result = await maybe_start_code_mode_fallback_retry(_workflow_run(**kwargs), "org_test")
-
- assert result is None
- run_workflow.assert_not_called()
diff --git a/tests/unit/test_credential_fallback.py b/tests/unit/test_credential_fallback.py
index f3c640a56..8d19986b8 100644
--- a/tests/unit/test_credential_fallback.py
+++ b/tests/unit/test_credential_fallback.py
@@ -846,7 +846,7 @@ async def test_mark_workflow_run_as_failed_does_not_schedule_before_cleanup() ->
service = WorkflowService()
workflow_run = _workflow_run(failure_category=[{"category": "AUTH_FAILURE"}])
service._update_workflow_run_status = AsyncMock(return_value=workflow_run) # type: ignore[method-assign]
- service._schedule_run_fallback_retries = MagicMock() # type: ignore[method-assign]
+ service._schedule_credential_fallback_retry = MagicMock() # type: ignore[method-assign]
result = await service.mark_workflow_run_as_failed(
workflow_run_id="wr_failed",
@@ -855,7 +855,7 @@ async def test_mark_workflow_run_as_failed_does_not_schedule_before_cleanup() ->
)
assert result == workflow_run
- service._schedule_run_fallback_retries.assert_not_called()
+ service._schedule_credential_fallback_retry.assert_not_called()
@pytest.mark.asyncio
@@ -866,7 +866,7 @@ async def test_mark_workflow_run_as_terminated_does_not_schedule_before_cleanup(
failure_category=[{"category": "AUTH_FAILURE"}],
)
service._update_workflow_run_status = AsyncMock(return_value=workflow_run) # type: ignore[method-assign]
- service._schedule_run_fallback_retries = MagicMock() # type: ignore[method-assign]
+ service._schedule_credential_fallback_retry = MagicMock() # type: ignore[method-assign]
result = await service.mark_workflow_run_as_terminated(
workflow_run_id="wr_failed",
@@ -875,7 +875,7 @@ async def test_mark_workflow_run_as_terminated_does_not_schedule_before_cleanup(
)
assert result == workflow_run
- service._schedule_run_fallback_retries.assert_not_called()
+ service._schedule_credential_fallback_retry.assert_not_called()
@pytest.mark.asyncio
@@ -883,7 +883,7 @@ async def test_clean_up_workflow_schedules_credential_fallback_retry() -> None:
service = WorkflowService()
workflow_run = _workflow_run(failure_category=[{"category": "AUTH_FAILURE"}])
workflow = _workflow([])
- service._schedule_run_fallback_retries = MagicMock() # type: ignore[method-assign]
+ service._schedule_credential_fallback_retry = MagicMock() # type: ignore[method-assign]
browser_cleanup_result = SimpleNamespace(
browser_state=None,
tasks=[],
@@ -909,7 +909,7 @@ async def test_clean_up_workflow_schedules_credential_fallback_retry() -> None:
browser_cleanup_result=browser_cleanup_result, # type: ignore[arg-type]
)
- service._schedule_run_fallback_retries.assert_called_once_with(workflow_run)
+ service._schedule_credential_fallback_retry.assert_called_once_with(workflow_run)
@pytest.mark.asyncio
@@ -919,7 +919,7 @@ async def test_clean_up_workflow_schedules_retry_even_when_webhook_raises() -> N
service = WorkflowService()
workflow_run = _workflow_run(failure_category=[{"category": "AUTH_FAILURE"}])
workflow = _workflow([])
- service._schedule_run_fallback_retries = MagicMock() # type: ignore[method-assign]
+ service._schedule_credential_fallback_retry = MagicMock() # type: ignore[method-assign]
service.execute_workflow_webhook = AsyncMock(side_effect=RuntimeError("webhook prep failed")) # type: ignore[method-assign]
browser_cleanup_result = SimpleNamespace(
browser_state=None,
@@ -947,7 +947,7 @@ async def test_clean_up_workflow_schedules_retry_even_when_webhook_raises() -> N
browser_cleanup_result=browser_cleanup_result, # type: ignore[arg-type]
)
- service._schedule_run_fallback_retries.assert_called_once_with(workflow_run)
+ service._schedule_credential_fallback_retry.assert_called_once_with(workflow_run)
@pytest.mark.asyncio
@@ -957,7 +957,7 @@ async def test_clean_up_workflow_schedules_retry_when_earlier_cleanup_step_raise
service = WorkflowService()
workflow_run = _workflow_run(failure_category=[{"category": "AUTH_FAILURE"}])
workflow = _workflow([])
- service._schedule_run_fallback_retries = MagicMock() # type: ignore[method-assign]
+ service._schedule_credential_fallback_retry = MagicMock() # type: ignore[method-assign]
browser_cleanup_result = SimpleNamespace(
browser_state=None,
tasks=[],
@@ -984,7 +984,7 @@ async def test_clean_up_workflow_schedules_retry_when_earlier_cleanup_step_raise
browser_cleanup_result=browser_cleanup_result, # type: ignore[arg-type]
)
- service._schedule_run_fallback_retries.assert_called_once_with(workflow_run)
+ service._schedule_credential_fallback_retry.assert_called_once_with(workflow_run)
@pytest.mark.asyncio
@@ -995,4 +995,4 @@ async def test_fallback_hook_exception_does_not_raise() -> None:
"skyvern.forge.sdk.workflow.service.maybe_start_credential_fallback_retry",
AsyncMock(side_effect=RuntimeError("boom")),
):
- await WorkflowService._start_run_fallback_retries_best_effort(workflow_run)
+ await WorkflowService._start_credential_fallback_retry_best_effort(workflow_run)
diff --git a/tests/unit/test_should_run_script.py b/tests/unit/test_should_run_script.py
index 9d712547e..f42573a71 100644
--- a/tests/unit/test_should_run_script.py
+++ b/tests/unit/test_should_run_script.py
@@ -45,11 +45,7 @@ def _make_workflow(
)
-def _make_run(
- run_with: str | None = "agent",
- workflow_run_id: str = "wr_test",
- retried_from_workflow_run_id: str | None = None,
-) -> WorkflowRun:
+def _make_run(run_with: str | None = "agent", workflow_run_id: str = "wr_test") -> WorkflowRun:
return WorkflowRun(
workflow_run_id=workflow_run_id,
workflow_id="wf_test",
@@ -57,7 +53,6 @@ def _make_run(
organization_id="org_test",
status=WorkflowRunStatus.running,
run_with=run_with,
- retried_from_workflow_run_id=retried_from_workflow_run_id,
created_at=datetime.now(timezone.utc),
modified_at=datetime.now(timezone.utc),
)
@@ -235,83 +230,6 @@ class TestCodeModeGate:
assert len(agent_function.calls) == 1
-class _UpgradingAgentFunction(AgentFunction):
- def __init__(self, upgrade: bool, keep_code: bool = True) -> None:
- self.upgrade = upgrade
- self.keep_code = keep_code
- self.upgrade_calls: list[tuple[str, str, str]] = []
-
- async def should_upgrade_to_code_mode(self, *, workflow: Workflow, workflow_run: WorkflowRun) -> bool:
- self.upgrade_calls.append(
- (workflow_run.workflow_run_id, workflow_run.organization_id, workflow.workflow_permanent_id)
- )
- return self.upgrade
-
- async def should_keep_code_mode_for_workflow_run(self, *, workflow: Workflow, workflow_run: WorkflowRun) -> bool:
- return self.keep_code
-
-
-class TestCodeModeUpgrade:
- """A flag-gated rollout can upgrade an agent-intended run to code."""
-
- @pytest.mark.asyncio
- async def test_upgrade_flips_agent_to_code(self, monkeypatch):
- agent_function = _UpgradingAgentFunction(upgrade=True, keep_code=True)
- monkeypatch.setattr(app, "AGENT_FUNCTION", agent_function)
- from skyvern.forge.sdk.workflow.service import WorkflowService
-
- wf = _make_workflow(run_with="agent")
- wr = _make_run(run_with="agent")
- assert await WorkflowService().should_run_script(wf, wr) is True
- assert agent_function.upgrade_calls == [("wr_test", "org_test", "wpid_test")]
-
- @pytest.mark.asyncio
- async def test_upgrade_disabled_stays_agent(self, monkeypatch):
- agent_function = _UpgradingAgentFunction(upgrade=False)
- monkeypatch.setattr(app, "AGENT_FUNCTION", agent_function)
- from skyvern.forge.sdk.workflow.service import WorkflowService
-
- wf = _make_workflow(run_with="agent")
- wr = _make_run(run_with="agent")
- assert await WorkflowService().should_run_script(wf, wr) is False
-
- @pytest.mark.asyncio
- async def test_upgraded_run_still_passes_keep_code_gate(self, monkeypatch):
- """Upgrade True but keep-code gate False → still agent (gate can veto)."""
- agent_function = _UpgradingAgentFunction(upgrade=True, keep_code=False)
- monkeypatch.setattr(app, "AGENT_FUNCTION", agent_function)
- from skyvern.forge.sdk.workflow.service import WorkflowService
-
- wf = _make_workflow(run_with="agent")
- wr = _make_run(run_with="agent")
- assert await WorkflowService().should_run_script(wf, wr) is False
-
- @pytest.mark.asyncio
- async def test_upgrade_not_consulted_when_already_code(self, monkeypatch):
- agent_function = _UpgradingAgentFunction(upgrade=False)
- monkeypatch.setattr(app, "AGENT_FUNCTION", agent_function)
- from skyvern.forge.sdk.workflow.service import WorkflowService
-
- wf = _make_workflow(run_with="code")
- wr = _make_run(run_with="code")
- assert await WorkflowService().should_run_script(wf, wr) is True
- assert agent_function.upgrade_calls == []
-
- @pytest.mark.asyncio
- async def test_fallback_retry_never_re_upgraded(self, monkeypatch):
- # A code->agent fallback retry (run_with=agent, retried_from set) must never be re-upgraded
- # to code, or the rollout would re-run the very script that just failed and the fallback
- # would be a no-op for exactly the rollout population.
- agent_function = _UpgradingAgentFunction(upgrade=True, keep_code=True)
- monkeypatch.setattr(app, "AGENT_FUNCTION", agent_function)
- from skyvern.forge.sdk.workflow.service import WorkflowService
-
- wf = _make_workflow(run_with="agent")
- wr = _make_run(run_with="agent", retried_from_workflow_run_id="wr_orig")
- assert await WorkflowService().should_run_script(wf, wr) is False
- assert agent_function.upgrade_calls == []
-
-
class TestScriptReviewerGate:
"""The script reviewer should only fire when the script was actually executed.