Skip to content

Automate browser based workflows with AI

Refuse out-of-credit work at run start, code blocks, cached steps and SDK actions (SKY-18216) (#8894)

Andrew Neilsonfcf65e0882ad
diff --git a/skyvern/forge/sdk/routes/sdk.py b/skyvern/forge/sdk/routes/sdk.py
index 276e4a108..397015233 100644
--- a/skyvern/forge/sdk/routes/sdk.py
+++ b/skyvern/forge/sdk/routes/sdk.py
@@ -11,6 +11,7 @@ from skyvern.exceptions import (
ScreenshotTargetClosed,
SkyvernActionFailed,
SkyvernHTTPException,
+ TaskTerminationError,
)
from skyvern.forge import app
from skyvern.forge.sdk.api.files import uploaded_file_id_for_local_uri, validate_download_url
@@ -81,6 +82,11 @@ async def run_sdk_action(
# The whole action runs on this process's event loop, so the cap is taken before any database
# write or page work: an organization over its cap costs a rejected request, not a started one.
async with app.RATE_LIMITER.limit_sdk_action_concurrency(organization.organization_id):
+ # Not the API permission checker: on older billing plans it demands a whole run's worth of credits.
+ try:
+ await app.AGENT_FUNCTION.validate_task_execution(organization_id=organization.organization_id)
+ except TaskTerminationError:
+ raise HTTPException(status_code=status.HTTP_402_PAYMENT_REQUIRED, detail="Not enough credit balance.")
return await _run_sdk_action(action_request, organization, created_by=user_id)
diff --git a/skyvern/forge/sdk/workflow/models/block.py b/skyvern/forge/sdk/workflow/models/block.py
index 42a0b7c6f..843930902 100644
--- a/skyvern/forge/sdk/workflow/models/block.py
+++ b/skyvern/forge/sdk/workflow/models/block.py
@@ -97,6 +97,7 @@ from skyvern.exceptions import (
AzureConfigurationError,
BlockedHost,
BlockedNavigationDestination,
+ BlockTerminationError,
BranchEvaluationContextTooLargeError,
CodeBlockRunnerSelectionError,
ConditionalBranchEvaluationError,
@@ -9292,6 +9293,23 @@ async def wrapper({default_args}):
**kwargs: dict,
) -> BlockResult:
await app.AGENT_FUNCTION.validate_code_block(organization_id=organization_id)
+ try:
+ await app.AGENT_FUNCTION.validate_block_execution(
+ block=self,
+ workflow_run_id=workflow_run_id,
+ workflow_run_block_id=workflow_run_block_id,
+ organization_id=organization_id,
+ )
+ except BlockTerminationError as e:
+ # Returned, not raised: execute() rewrites a raised error into a generic machinery failure.
+ return await self.build_block_result(
+ success=False,
+ failure_reason=get_user_facing_exception_message(e),
+ status=BlockStatus.failed,
+ workflow_run_block_id=workflow_run_block_id,
+ organization_id=organization_id,
+ can_continue_after_failure=False,
+ )
block_context = skyvern_context.current()
workflow_run_context = self.get_workflow_run_context(workflow_run_id)
diff --git a/skyvern/forge/sdk/workflow/service.py b/skyvern/forge/sdk/workflow/service.py
index 0ad45f827..e5b4bdf01 100644
--- a/skyvern/forge/sdk/workflow/service.py
+++ b/skyvern/forge/sdk/workflow/service.py
@@ -72,6 +72,7 @@ from skyvern.exceptions import (
SequentialCredentialLimitExceeded,
SkyvernException,
SkyvernHTTPException,
+ StepTerminationError,
UnrecognizedWorkflowParameters,
WorkflowAttemptDispatchSuperseded,
WorkflowHasNoBlocks,
@@ -6884,7 +6885,10 @@ class WorkflowService:
except WorkflowAttemptDispatchSuperseded:
raise
except Exception as error:
- LOG.exception("Workflow run admission failed", workflow_run_id=workflow_run_id)
+ if isinstance(error, SkyvernHTTPException) and error.status_code == 402:
+ LOG.info("Workflow run refused at admission", workflow_run_id=workflow_run_id, reason=str(error))
+ else:
+ LOG.exception("Workflow run admission failed", workflow_run_id=workflow_run_id)
failed_run = await self._mark_workflow_run_as_failed_for_dispatch(
workflow_run_id=workflow_run_id,
organization_id=organization_id,
@@ -9243,7 +9247,8 @@ class WorkflowService:
duration_ms=block_exec_duration_ms,
)
block_executed_with_code = False
- except _WorkflowRunDispatchScopeError:
+ # A refused step is not a script failure: AI fallback would re-run the block into the same refusal.
+ except (_WorkflowRunDispatchScopeError, StepTerminationError):
raise
except Exception as e:
block_exec_duration_ms = round((time.monotonic() - block_exec_start) * 1000, 1)
diff --git a/skyvern/services/script_service.py b/skyvern/services/script_service.py
index ecee2a295..b91a81928 100644
--- a/skyvern/services/script_service.py
+++ b/skyvern/services/script_service.py
@@ -36,6 +36,7 @@ from skyvern.exceptions import (
InProcessScriptExecutionDenied,
ScriptNotFound,
ScriptTerminationException,
+ StepTerminationError,
WorkflowRunNotFound,
)
from skyvern.forge import app
@@ -783,6 +784,21 @@ async def _create_workflow_block_run_and_task(
admitted = await app.AGENT_FUNCTION.admit_recipe_step_attempt(task, step, is_cached=True)
if not admitted:
raise RuntimeError("Recipe step was not admitted")
+ else:
+ try:
+ await app.AGENT_FUNCTION.validate_step_execution(task, step)
+ except StepTerminationError as e:
+ await _update_workflow_block(
+ workflow_run_block_id,
+ BlockStatus.failed,
+ task_id=task_id,
+ task_status=TaskStatus.failed,
+ failure_reason=e.message,
+ step_id=step_id,
+ step_status=StepStatus.failed,
+ label=label,
+ )
+ raise
# The agent path settles a prior failed block's owned capture in execute_safe; a cached
# successor reaches the same browser here, so it settles it before its own reads. Kept above
diff --git a/tests/unit/test_code_block_downloads.py b/tests/unit/test_code_block_downloads.py
index 09df21738..580c1daf3 100644
--- a/tests/unit/test_code_block_downloads.py
+++ b/tests/unit/test_code_block_downloads.py
@@ -271,6 +271,7 @@ def _fake_storage_app(
),
AGENT_FUNCTION=SimpleNamespace(
validate_code_block=AsyncMock(),
+ validate_block_execution=AsyncMock(),
# Secure CodeBlock runner gating — match the OSS base no-op so execute() runs legacy.
should_use_codeblock_runner=AsyncMock(return_value=False),
execute_code_block_override=AsyncMock(return_value=None),
diff --git a/tests/unit/test_run_sdk_action_failures.py b/tests/unit/test_run_sdk_action_failures.py
index 76847b43c..08712723d 100644
--- a/tests/unit/test_run_sdk_action_failures.py
+++ b/tests/unit/test_run_sdk_action_failures.py
@@ -17,6 +17,7 @@ from skyvern.exceptions import (
ScrapingFailed,
ScreenshotTargetClosed,
SkyvernActionFailed,
+ TaskTerminationError,
)
from skyvern.forge import app as forge_app
from skyvern.forge.sdk.artifact.storage.local import LocalStorage
@@ -56,6 +57,7 @@ def mock_organization() -> Any:
@pytest.fixture
def mock_app() -> Any:
app = MagicMock()
+ app.AGENT_FUNCTION.validate_task_execution = AsyncMock()
workflow_run = MagicMock(workflow_run_id="wr_test", workflow_id="w_test")
workflow = MagicMock(workflow_id="w_test", workflow_permanent_id="wpid_test", title="t")
task = MagicMock(task_id="tsk_test", organization_id="o_test", max_screenshot_scrolls=None)
@@ -830,6 +832,22 @@ async def test_action_over_the_concurrency_cap_is_rejected_before_any_run_is_cre
await run_sdk_action(mock_request, organization=mock_organization)
assert rejected.value.status_code == 429
+ mock_app.AGENT_FUNCTION.validate_task_execution.assert_not_awaited()
mock_app.WORKFLOW_SERVICE.create_empty_workflow.assert_not_awaited()
mock_app.DATABASE.tasks.create_task.assert_not_awaited()
mock_app.DATABASE.tasks.create_step.assert_not_awaited()
+
+
+async def test_credit_refusal_creates_no_task(mock_request: Any, mock_organization: Any, mock_app: Any) -> None:
+ mock_app.AGENT_FUNCTION.validate_task_execution = AsyncMock(
+ side_effect=TaskTerminationError(reason="Not enough credit balance.")
+ )
+ with (
+ patch("skyvern.forge.sdk.routes.sdk.app", mock_app),
+ pytest.raises(HTTPException) as exc_info,
+ ):
+ await run_sdk_action(mock_request, organization=mock_organization)
+
+ assert exc_info.value.status_code == 402
+ mock_app.DATABASE.tasks.create_task.assert_not_awaited()
diff --git a/tests/unit/test_workflow_block_engine.py b/tests/unit/test_workflow_block_engine.py
index d5b76f714..6bc6047e3 100644
--- a/tests/unit/test_workflow_block_engine.py
+++ b/tests/unit/test_workflow_block_engine.py
@@ -1018,6 +1018,7 @@ async def test_script_create_site_stamps_the_script_run_marker(scoped_context: S
AsyncMock()
)
mock_app.AGENT_FUNCTION.is_recipe_step_attempt = AsyncMock(return_value=False)
+ mock_app.AGENT_FUNCTION.validate_step_execution = AsyncMock()
mock_app.DATABASE.tasks.update_step = AsyncMock(return_value=SimpleNamespace(step_id="stp_script_marker"))
block_id, task_id, step_id = await script_service._create_workflow_block_run_and_task(
diff --git a/tests/unit/workflow/test_execute_single_block_phases.py b/tests/unit/workflow/test_execute_single_block_phases.py
index c12950377..faaeed8c5 100644
--- a/tests/unit/workflow/test_execute_single_block_phases.py
+++ b/tests/unit/workflow/test_execute_single_block_phases.py
@@ -19,6 +19,7 @@ from unittest.mock import AsyncMock, MagicMock, patch
import pytest
from structlog.testing import capture_logs
+from skyvern.exceptions import StepTerminationError
from skyvern.forge import app
from skyvern.forge.sdk.db.enums import BrowserSeedSource
from skyvern.forge.sdk.workflow.context_manager import BlockOutcome, WorkflowRunContext
@@ -725,6 +726,34 @@ async def test_ai_fallback_disabled_keeps_script_failure(monkeypatch: pytest.Mon
assert should_stop is True
+async def test_refused_script_step_fails_the_run_without_ai_fallback(monkeypatch: pytest.MonkeyPatch) -> None:
+ service = WorkflowService()
+ block = _navigation_block("cached_nav")
+ execute_safe = AsyncMock()
+ monkeypatch.setattr(NavigationBlock, "execute_safe", execute_safe)
+ monkeypatch.setattr(NavigationBlock, "_apply_workflow_system_prompt", lambda self, ctx: None)
+ failed_run = MagicMock()
+ mark_failed = AsyncMock(return_value=failed_run)
+ monkeypatch.setattr(WorkflowService, "mark_workflow_run_as_failed_if_not_final", mark_failed)
+
+ async def run_cached_task() -> None:
+ raise StepTerminationError(step_id="stp_1", reason="Not enough credit balance.")
+
+ workflow_run, _, _, should_stop, _ = await _run_single_block(
+ service,
+ block,
+ is_script_run=True,
+ script_blocks_by_label={"cached_nav": _script_block("cached_nav", "await run_cached_task()")},
+ loaded_script_module=SimpleNamespace(run_cached_task=run_cached_task),
+ )
+
+ execute_safe.assert_not_awaited()
+ assert "Not enough credit balance." in mark_failed.await_args.kwargs["failure_reason"]
+ assert workflow_run is failed_run
+ assert should_stop is True
+
+
@pytest.mark.asyncio
async def test_script_success_skips_agent_execution(monkeypatch: pytest.MonkeyPatch) -> None:
service = WorkflowService()