Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
Refuse out-of-credit work at run start, code blocks, cached steps and SDK actions (SKY-18216) (#8894)
Andrew Neilsonfcf65e0882ad
diff --git a/skyvern/forge/sdk/routes/sdk.py b/skyvern/forge/sdk/routes/sdk.pyindex 276e4a108..397015233 100644--- a/skyvern/forge/sdk/routes/sdk.py+++ b/skyvern/forge/sdk/routes/sdk.py@@ -11,6 +11,7 @@ from skyvern.exceptions import (ScreenshotTargetClosed,SkyvernActionFailed,SkyvernHTTPException,+ TaskTerminationError,)from skyvern.forge import appfrom skyvern.forge.sdk.api.files import uploaded_file_id_for_local_uri, validate_download_url@@ -81,6 +82,11 @@ async def run_sdk_action(# The whole action runs on this process's event loop, so the cap is taken before any database# write or page work: an organization over its cap costs a rejected request, not a started one.async with app.RATE_LIMITER.limit_sdk_action_concurrency(organization.organization_id):+ # Not the API permission checker: on older billing plans it demands a whole run's worth of credits.+ try:+ await app.AGENT_FUNCTION.validate_task_execution(organization_id=organization.organization_id)+ except TaskTerminationError:+ raise HTTPException(status_code=status.HTTP_402_PAYMENT_REQUIRED, detail="Not enough credit balance.")return await _run_sdk_action(action_request, organization, created_by=user_id)diff --git a/skyvern/forge/sdk/workflow/models/block.py b/skyvern/forge/sdk/workflow/models/block.pyindex 42a0b7c6f..843930902 100644--- a/skyvern/forge/sdk/workflow/models/block.py+++ b/skyvern/forge/sdk/workflow/models/block.py@@ -97,6 +97,7 @@ from skyvern.exceptions import (AzureConfigurationError,BlockedHost,BlockedNavigationDestination,+ BlockTerminationError,BranchEvaluationContextTooLargeError,CodeBlockRunnerSelectionError,ConditionalBranchEvaluationError,@@ -9292,6 +9293,23 @@ async def wrapper({default_args}):**kwargs: dict,) -> BlockResult:await app.AGENT_FUNCTION.validate_code_block(organization_id=organization_id)+ try:+ await app.AGENT_FUNCTION.validate_block_execution(+ block=self,+ workflow_run_id=workflow_run_id,+ workflow_run_block_id=workflow_run_block_id,+ organization_id=organization_id,+ )+ except BlockTerminationError as e:+ # Returned, not raised: execute() rewrites a raised error into a generic machinery failure.+ return await self.build_block_result(+ success=False,+ failure_reason=get_user_facing_exception_message(e),+ status=BlockStatus.failed,+ workflow_run_block_id=workflow_run_block_id,+ organization_id=organization_id,+ can_continue_after_failure=False,+ )block_context = skyvern_context.current()workflow_run_context = self.get_workflow_run_context(workflow_run_id)diff --git a/skyvern/forge/sdk/workflow/service.py b/skyvern/forge/sdk/workflow/service.pyindex 0ad45f827..e5b4bdf01 100644--- a/skyvern/forge/sdk/workflow/service.py+++ b/skyvern/forge/sdk/workflow/service.py@@ -72,6 +72,7 @@ from skyvern.exceptions import (SequentialCredentialLimitExceeded,SkyvernException,SkyvernHTTPException,+ StepTerminationError,UnrecognizedWorkflowParameters,WorkflowAttemptDispatchSuperseded,WorkflowHasNoBlocks,@@ -6884,7 +6885,10 @@ class WorkflowService:except WorkflowAttemptDispatchSuperseded:raiseexcept Exception as error:- LOG.exception("Workflow run admission failed", workflow_run_id=workflow_run_id)+ if isinstance(error, SkyvernHTTPException) and error.status_code == 402:+ LOG.info("Workflow run refused at admission", workflow_run_id=workflow_run_id, reason=str(error))+ else:+ LOG.exception("Workflow run admission failed", workflow_run_id=workflow_run_id)failed_run = await self._mark_workflow_run_as_failed_for_dispatch(workflow_run_id=workflow_run_id,organization_id=organization_id,@@ -9243,7 +9247,8 @@ class WorkflowService:duration_ms=block_exec_duration_ms,)block_executed_with_code = False- except _WorkflowRunDispatchScopeError:+ # A refused step is not a script failure: AI fallback would re-run the block into the same refusal.+ except (_WorkflowRunDispatchScopeError, StepTerminationError):raiseexcept Exception as e:block_exec_duration_ms = round((time.monotonic() - block_exec_start) * 1000, 1)diff --git a/skyvern/services/script_service.py b/skyvern/services/script_service.pyindex ecee2a295..b91a81928 100644--- a/skyvern/services/script_service.py+++ b/skyvern/services/script_service.py@@ -36,6 +36,7 @@ from skyvern.exceptions import (InProcessScriptExecutionDenied,ScriptNotFound,ScriptTerminationException,+ StepTerminationError,WorkflowRunNotFound,)from skyvern.forge import app@@ -783,6 +784,21 @@ async def _create_workflow_block_run_and_task(admitted = await app.AGENT_FUNCTION.admit_recipe_step_attempt(task, step, is_cached=True)if not admitted:raise RuntimeError("Recipe step was not admitted")+ else:+ try:+ await app.AGENT_FUNCTION.validate_step_execution(task, step)+ except StepTerminationError as e:+ await _update_workflow_block(+ workflow_run_block_id,+ BlockStatus.failed,+ task_id=task_id,+ task_status=TaskStatus.failed,+ failure_reason=e.message,+ step_id=step_id,+ step_status=StepStatus.failed,+ label=label,+ )+ raise# The agent path settles a prior failed block's owned capture in execute_safe; a cached# successor reaches the same browser here, so it settles it before its own reads. Kept abovediff --git a/tests/unit/test_code_block_downloads.py b/tests/unit/test_code_block_downloads.pyindex 09df21738..580c1daf3 100644--- a/tests/unit/test_code_block_downloads.py+++ b/tests/unit/test_code_block_downloads.py@@ -271,6 +271,7 @@ def _fake_storage_app(),AGENT_FUNCTION=SimpleNamespace(validate_code_block=AsyncMock(),+ validate_block_execution=AsyncMock(),# Secure CodeBlock runner gating — match the OSS base no-op so execute() runs legacy.should_use_codeblock_runner=AsyncMock(return_value=False),execute_code_block_override=AsyncMock(return_value=None),diff --git a/tests/unit/test_run_sdk_action_failures.py b/tests/unit/test_run_sdk_action_failures.pyindex 76847b43c..08712723d 100644--- a/tests/unit/test_run_sdk_action_failures.py+++ b/tests/unit/test_run_sdk_action_failures.py@@ -17,6 +17,7 @@ from skyvern.exceptions import (ScrapingFailed,ScreenshotTargetClosed,SkyvernActionFailed,+ TaskTerminationError,)from skyvern.forge import app as forge_appfrom skyvern.forge.sdk.artifact.storage.local import LocalStorage@@ -56,6 +57,7 @@ def mock_organization() -> Any:@pytest.fixturedef mock_app() -> Any:app = MagicMock()+ app.AGENT_FUNCTION.validate_task_execution = AsyncMock()workflow_run = MagicMock(workflow_run_id="wr_test", workflow_id="w_test")workflow = MagicMock(workflow_id="w_test", workflow_permanent_id="wpid_test", title="t")task = MagicMock(task_id="tsk_test", organization_id="o_test", max_screenshot_scrolls=None)@@ -830,6 +832,22 @@ async def test_action_over_the_concurrency_cap_is_rejected_before_any_run_is_creawait run_sdk_action(mock_request, organization=mock_organization)assert rejected.value.status_code == 429+ mock_app.AGENT_FUNCTION.validate_task_execution.assert_not_awaited()mock_app.WORKFLOW_SERVICE.create_empty_workflow.assert_not_awaited()mock_app.DATABASE.tasks.create_task.assert_not_awaited()mock_app.DATABASE.tasks.create_step.assert_not_awaited()+++async def test_credit_refusal_creates_no_task(mock_request: Any, mock_organization: Any, mock_app: Any) -> None:+ mock_app.AGENT_FUNCTION.validate_task_execution = AsyncMock(+ side_effect=TaskTerminationError(reason="Not enough credit balance.")+ )+ with (+ patch("skyvern.forge.sdk.routes.sdk.app", mock_app),+ pytest.raises(HTTPException) as exc_info,+ ):+ await run_sdk_action(mock_request, organization=mock_organization)++ assert exc_info.value.status_code == 402+ mock_app.DATABASE.tasks.create_task.assert_not_awaited()diff --git a/tests/unit/test_workflow_block_engine.py b/tests/unit/test_workflow_block_engine.pyindex d5b76f714..6bc6047e3 100644--- a/tests/unit/test_workflow_block_engine.py+++ b/tests/unit/test_workflow_block_engine.py@@ -1018,6 +1018,7 @@ async def test_script_create_site_stamps_the_script_run_marker(scoped_context: SAsyncMock())mock_app.AGENT_FUNCTION.is_recipe_step_attempt = AsyncMock(return_value=False)+ mock_app.AGENT_FUNCTION.validate_step_execution = AsyncMock()mock_app.DATABASE.tasks.update_step = AsyncMock(return_value=SimpleNamespace(step_id="stp_script_marker"))block_id, task_id, step_id = await script_service._create_workflow_block_run_and_task(diff --git a/tests/unit/workflow/test_execute_single_block_phases.py b/tests/unit/workflow/test_execute_single_block_phases.pyindex c12950377..faaeed8c5 100644--- a/tests/unit/workflow/test_execute_single_block_phases.py+++ b/tests/unit/workflow/test_execute_single_block_phases.py@@ -19,6 +19,7 @@ from unittest.mock import AsyncMock, MagicMock, patchimport pytestfrom structlog.testing import capture_logs+from skyvern.exceptions import StepTerminationErrorfrom skyvern.forge import appfrom skyvern.forge.sdk.db.enums import BrowserSeedSourcefrom skyvern.forge.sdk.workflow.context_manager import BlockOutcome, WorkflowRunContext@@ -725,6 +726,34 @@ async def test_ai_fallback_disabled_keeps_script_failure(monkeypatch: pytest.Monassert should_stop is True+async def test_refused_script_step_fails_the_run_without_ai_fallback(monkeypatch: pytest.MonkeyPatch) -> None:+ service = WorkflowService()+ block = _navigation_block("cached_nav")+ execute_safe = AsyncMock()+ monkeypatch.setattr(NavigationBlock, "execute_safe", execute_safe)+ monkeypatch.setattr(NavigationBlock, "_apply_workflow_system_prompt", lambda self, ctx: None)+ failed_run = MagicMock()+ mark_failed = AsyncMock(return_value=failed_run)+ monkeypatch.setattr(WorkflowService, "mark_workflow_run_as_failed_if_not_final", mark_failed)++ async def run_cached_task() -> None:+ raise StepTerminationError(step_id="stp_1", reason="Not enough credit balance.")++ workflow_run, _, _, should_stop, _ = await _run_single_block(+ service,+ block,+ is_script_run=True,+ script_blocks_by_label={"cached_nav": _script_block("cached_nav", "await run_cached_task()")},+ loaded_script_module=SimpleNamespace(run_cached_task=run_cached_task),+ )++ execute_safe.assert_not_awaited()+ assert "Not enough credit balance." in mark_failed.await_args.kwargs["failure_reason"]+ assert workflow_run is failed_run+ assert should_stop is True++@pytest.mark.asyncioasync def test_script_success_skips_agent_execution(monkeypatch: pytest.MonkeyPatch) -> None:service = WorkflowService()