Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
fix(SKY-9568): persist block status on script termination, distinguish complete-reject from intentional terminate (#5862)
pedrohsdbdddbe89fb754
diff --git a/skyvern/core/script_generations/script_skyvern_page.py b/skyvern/core/script_generations/script_skyvern_page.pyindex 47418bb4d..a2d4c8059 100644--- a/skyvern/core/script_generations/script_skyvern_page.py+++ b/skyvern/core/script_generations/script_skyvern_page.py@@ -19,7 +19,7 @@ from skyvern.core.script_generations.real_skyvern_page_ai import RealSkyvernPagefrom skyvern.core.script_generations.skyvern_page import ActionCall, ActionMetadata, RunContext, SkyvernPagefrom skyvern.core.script_generations.skyvern_page_ai import SkyvernPageAifrom skyvern.errors.errors import UserDefinedError-from skyvern.exceptions import ScriptTerminationException, WorkflowRunNotFound+from skyvern.exceptions import IllegitCompleteScriptTermination, ScriptTerminationException, WorkflowRunNotFoundfrom skyvern.forge import appfrom skyvern.forge.prompts import prompt_enginefrom skyvern.forge.sdk.api.files import (@@ -1076,7 +1076,9 @@ class ScriptSkyvernPage(SkyvernPage):# result = await ActionHandler.handle_action(self.scraped_page, task, step, self.page, action)result = await handle_complete_action(action, self.page, self.scraped_page, task, step)if result and result[-1].success is False:- raise ScriptTerminationException(result[-1].exception_message)+ # Coerce empty/None messages so downstream str(e) is meaningful.+ msg = result[-1].exception_message or "complete-verify rejected without a message"+ raise IllegitCompleteScriptTermination(msg)# Capture final full-page screenshot at block completionawait self._create_final_screenshot()diff --git a/skyvern/exceptions.py b/skyvern/exceptions.pyindex f39cfc64a..244fead42 100644--- a/skyvern/exceptions.py+++ b/skyvern/exceptions.py@@ -1143,6 +1143,10 @@ class ScriptTerminationException(SkyvernException):super().__init__(reason)+class IllegitCompleteScriptTermination(ScriptTerminationException):+ """Raised when a cached script's page.complete() is rejected by the verifier; distinct from plain ScriptTerminationException, which is an intentional terminate()."""++class InvalidSchemaError(SkyvernException):def __init__(self, message: str, validation_errors: list[str] | None = None):self.message = messagediff --git a/skyvern/forge/sdk/workflow/service.py b/skyvern/forge/sdk/workflow/service.pyindex 402510249..d3c181510 100644--- a/skyvern/forge/sdk/workflow/service.py+++ b/skyvern/forge/sdk/workflow/service.py@@ -2406,6 +2406,10 @@ class WorkflowService:LOG.debug("Executing run_signature wrapper", wrapper_code=wrapper_code)+ # Pre-init so the success log can reference output_value+ # when ScriptTerminationException was caught (terminated+ # is now in script_success_statuses).+ output_value: Any = Nonetry:exec_code = compile(wrapper_code, "<run_signature>", "exec")exec(exec_code, exec_globals)@@ -2433,10 +2437,10 @@ class WorkflowService:status=BlockStatus(latest_block.status) if latest_block.status else BlockStatus.failed,workflow_run_block_id=latest_block.workflow_run_block_id,)- # Terminated is a valid script outcome when generate_script_on_terminal is set- script_success_statuses = {BlockStatus.completed}- if workflow.generate_script_on_terminal:- script_success_statuses.add(BlockStatus.terminated)+ # Terminate() is an explicit signal to stop, not a+ # failure to retry. `generate_script_on_terminal` is+ # orthogonal — it gates script generation, not fallback.+ script_success_statuses = {BlockStatus.completed, BlockStatus.terminated}block_exec_duration_ms = round((time.monotonic() - block_exec_start) * 1000, 1)if workflow_run_block_result.status in script_success_statuses:@@ -2603,7 +2607,36 @@ class WorkflowService:# if the failure is code-fixable.if fallback_episode_id and workflow_run_block_result:try:+ # None = unknown count (taskless wrb / fetch error);+ # only a confirmed zero downgrades fallback_succeeded.+ fallback_wrb_id = workflow_run_block_result.workflow_run_block_id+ agent_action_count: int | None = None+ action_summaries: list[dict] | None = None+ if fallback_wrb_id:+ try:+ wrb = await app.DATABASE.observer.get_workflow_run_block(+ workflow_run_block_id=fallback_wrb_id,+ organization_id=organization_id,+ )+ if wrb and wrb.task_id:+ actions = await app.DATABASE.tasks.get_task_actions(+ task_id=wrb.task_id,+ organization_id=organization_id,+ )+ agent_action_count = len(actions)+ action_summaries = build_action_summaries_with_timing(actions)+ except Exception:+ LOG.debug(+ "Could not fetch rich actions for fallback episode",+ fallback_wrb_id=fallback_wrb_id,+ exc_info=True,+ )++ # `completed` with confirmed 0 actions = the agent's+ # complete-verify accepting what the script's rejected.fallback_succeeded = workflow_run_block_result.status == BlockStatus.completed+ if fallback_succeeded and agent_action_count == 0:+ fallback_succeeded = False# Build agent actions summary for both success and failureagent_actions_summary: dict = {@@ -2614,42 +2647,24 @@ class WorkflowService:}if form_fields_for_episode:agent_actions_summary["form_fields"] = form_fields_for_episode+ if action_summaries is not None:+ agent_actions_summary["actions"] = action_summaries- # For failed fallbacks, capture the failure reasonif not fallback_succeeded:- agent_actions_summary["failure_reason"] = (- str(workflow_run_block_result.failure_reason)[:2000]- if workflow_run_block_result.failure_reason- else None- )+ if workflow_run_block_result.failure_reason:+ agent_actions_summary["failure_reason"] = str(workflow_run_block_result.failure_reason)[+ :2000+ ]+ elif workflow_run_block_result.status == BlockStatus.completed and agent_action_count == 0:+ agent_actions_summary["failure_reason"] = script_service.VERIFIER_SWAP_FAILURE_REASONLOG.info("AI fallback failed, keeping episode for triage",episode_id=fallback_episode_id,block_status=workflow_run_block_result.status,block_label=block.label,+ agent_action_count=agent_action_count,)- # Fetch rich action details from the fallback execution- fallback_wrb_id = workflow_run_block_result.workflow_run_block_id- if fallback_wrb_id:- try:- wrb = await app.DATABASE.observer.get_workflow_run_block(- workflow_run_block_id=fallback_wrb_id,- organization_id=organization_id,- )- if wrb and wrb.task_id:- actions = await app.DATABASE.tasks.get_task_actions(- task_id=wrb.task_id,- organization_id=organization_id,- )- agent_actions_summary["actions"] = build_action_summaries_with_timing(actions)- except Exception:- LOG.debug(- "Could not fetch rich actions for fallback episode",- fallback_wrb_id=fallback_wrb_id,- exc_info=True,- )-await app.DATABASE.scripts.update_fallback_episode(episode_id=fallback_episode_id,organization_id=organization_id,diff --git a/skyvern/services/script_service.py b/skyvern/services/script_service.pyindex 8579f994d..a260793a3 100644--- a/skyvern/services/script_service.py+++ b/skyvern/services/script_service.py@@ -29,6 +29,7 @@ from skyvern.core.script_generations.script_skyvern_page import script_run_contefrom skyvern.errors.errors import UserDefinedError, filter_to_user_defined_codesfrom skyvern.exceptions import (CachedDownloadError,+ IllegitCompleteScriptTermination,ScriptNotFound,ScriptTerminationException,StepTerminationError,@@ -91,6 +92,16 @@ from skyvern.webeye.scraper.scraped_page import ElementTreeFormatLOG = structlog.get_logger()jinja_sandbox_env = SandboxedEnvironment()+# Synthetic failure_reason recorded on a fallback episode when the AI fallback+# ended `completed` with zero actions taken — i.e. the AI's complete-verify+# accepted what the script's complete-verify rejected, so the agent didn't+# actually do anything. Shared between the two episode writers+# (script_service._fallback_to_ai_run + workflow/service.py:_execute_single_block)+# so they stay in sync.+VERIFIER_SWAP_FAILURE_REASON = (+ "AI fallback ended completed with 0 actions — complete-verify accepted what the script's complete-verify rejected"+)+# Max wait for any download signal after a cached click; downstream# .crdownload polling handles in-progress completion separately. (SKY-9431)CACHED_DOWNLOAD_NO_FILE_GRACE_SECONDS = 60@@ -623,6 +634,43 @@ async def _record_output_parameter_value()+async def _handle_script_termination(+ e: ScriptTerminationException,+ block_kind: str,+ workflow_run_block_id: str | None,+ task_id: str | None,+ step_id: str | None,+ cache_key: str,+) -> None:+ """Persist the block as failed (verifier-rejected complete) or terminated (intentional terminate).++ StepStatus uses `failed` for both: the enum has no `terminated` value (only created/running/failed/completed/canceled), and `failed` is the existing codebase convention for non-completed steps.+ """+ if isinstance(e, IllegitCompleteScriptTermination):+ LOG.info("Script complete() rejected by verifier", block_kind=block_kind, cache_key=cache_key)+ block_status = BlockStatus.failed+ task_status = TaskStatus.failed+ else:+ LOG.info(+ "Script requested termination, not falling back to AI",+ block_kind=block_kind,+ cache_key=cache_key,+ )+ block_status = BlockStatus.terminated+ task_status = TaskStatus.terminated+ if workflow_run_block_id:+ await _update_workflow_block(+ workflow_run_block_id,+ block_status,+ task_id=task_id,+ task_status=task_status,+ step_id=step_id,+ step_status=StepStatus.failed,+ label=cache_key,+ failure_reason=str(e),+ )++async def _update_workflow_block(workflow_run_block_id: str,status: BlockStatus,@@ -1402,23 +1450,42 @@ async def _fallback_to_ai_run(# Update fallback episode with AI execution resultsif fallback_episode_id:try:- fallback_succeeded = task.status not in [TaskStatus.terminated, TaskStatus.failed]- agent_actions_summary: dict[str, Any] = {- "block_status": str(task.status),- }- if form_fields_snapshot:- agent_actions_summary["form_fields"] = form_fields_snapshot- if not fallback_succeeded and task.failure_reason:- agent_actions_summary["failure_reason"] = str(task.failure_reason)[:2000]+ # AI runs on the existing task and may retry across steps+ # (agent.execute_step recurses on next_step). Exclude the+ # script's pre-fallback step only — retry-chain actions are+ # counted. None = unknown (fetch error).+ agent_action_count: int | None = None+ action_summaries: list[dict] | None = Nonetry:- actions = await app.DATABASE.tasks.get_task_actions(+ all_actions = await app.DATABASE.tasks.get_task_actions(task_id=task_id,organization_id=organization_id,)- agent_actions_summary["actions"] = build_action_summaries_with_timing(actions)+ ai_actions = [a for a in all_actions if a.step_id != script_step_id]+ agent_action_count = len(ai_actions)+ action_summaries = build_action_summaries_with_timing(ai_actions)except Exception:LOG.debug("Could not fetch actions for fallback episode", exc_info=True)+ # Mirrors workflow/service.py's success predicate so the two+ # episode writers label the same outcome identically.+ fallback_succeeded = task.status == TaskStatus.completed+ if fallback_succeeded and agent_action_count == 0:+ fallback_succeeded = False++ agent_actions_summary: dict[str, Any] = {+ "block_status": str(task.status),+ }+ if form_fields_snapshot:+ agent_actions_summary["form_fields"] = form_fields_snapshot+ if action_summaries is not None:+ agent_actions_summary["actions"] = action_summaries+ if not fallback_succeeded:+ if task.failure_reason:+ agent_actions_summary["failure_reason"] = str(task.failure_reason)[:2000]+ elif task.status == TaskStatus.completed and agent_action_count == 0:+ agent_actions_summary["failure_reason"] = VERIFIER_SWAP_FAILURE_REASON+await app.DATABASE.scripts.update_fallback_episode(episode_id=fallback_episode_id,organization_id=organization_id,@@ -1809,10 +1876,8 @@ async def run_task()return output- except ScriptTerminationException:- # Explicit termination from script (e.g., target data does not exist).- # Do NOT fall back to AI — the script intentionally terminated.- LOG.info("Script requested termination. Not falling back to AI.", cache_key=cache_key)+ except ScriptTerminationException as e:+ await _handle_script_termination(e, "task block", workflow_run_block_id, task_id, step_id, cache_key)raiseexcept Exception as e:LOG.exception("Failed to run task block. Falling back to AI run.")@@ -2093,10 +2158,8 @@ async def download(label=cache_key,)- except ScriptTerminationException:- # Explicit termination from script (e.g., target data does not exist).- # Do NOT fall back to AI — the script intentionally terminated.- LOG.info("Script requested termination in download block. Not falling back to AI.", cache_key=cache_key)+ except ScriptTerminationException as e:+ await _handle_script_termination(e, "download block", workflow_run_block_id, task_id, step_id, cache_key)raiseexcept Exception as e:LOG.exception("Failed to run download block. Falling back to AI run.")@@ -2185,10 +2248,8 @@ async def action(label=cache_key,)- except ScriptTerminationException:- # Explicit termination from script (e.g., target data does not exist).- # Do NOT fall back to AI — the script intentionally terminated.- LOG.info("Script requested termination in action block. Not falling back to AI.", cache_key=cache_key)+ except ScriptTerminationException as e:+ await _handle_script_termination(e, "action block", workflow_run_block_id, task_id, step_id, cache_key)raiseexcept Exception as e:LOG.exception("Failed to run action block. Falling back to AI run.")@@ -2280,10 +2341,8 @@ async def login(label=cache_key,)- except ScriptTerminationException:- # Explicit termination from script (e.g., target data does not exist).- # Do NOT fall back to AI — the script intentionally terminated.- LOG.info("Script requested termination in login block. Not falling back to AI.", cache_key=cache_key)+ except ScriptTerminationException as e:+ await _handle_script_termination(e, "login block", workflow_run_block_id, task_id, step_id, cache_key)raiseexcept Exception as e:LOG.exception("Failed to run login block")diff --git a/tests/unit/test_handle_script_termination.py b/tests/unit/test_handle_script_termination.pynew file mode 100644index 000000000..cbe03bfbc--- /dev/null+++ b/tests/unit/test_handle_script_termination.py@@ -0,0 +1,84 @@+"""Locks the bookkeeping contract for _handle_script_termination (SKY-9568):+IllegitCompleteScriptTermination -> BlockStatus.failed (so AI fallback fires);+plain ScriptTerminationException -> BlockStatus.terminated (no fallback)."""++from unittest.mock import AsyncMock, patch++import pytest++from skyvern.exceptions import (+ IllegitCompleteScriptTermination,+ ScriptTerminationException,+)+from skyvern.forge.sdk.models import StepStatus+from skyvern.forge.sdk.schemas.tasks import TaskStatus+from skyvern.schemas.workflows import BlockStatus+from skyvern.services.script_service import _handle_script_termination+++async def test_helper_writes_failed_for_illegit_complete():+ e = IllegitCompleteScriptTermination("Illegit complete, data={'error': '...'}")+ with patch(+ "skyvern.services.script_service._update_workflow_block",+ new_callable=AsyncMock,+ ) as mock_update:+ await _handle_script_termination(+ e,+ "task block",+ workflow_run_block_id="wrb_1",+ task_id="tsk_1",+ step_id="stp_1",+ cache_key="MyTaskBlock",+ )+ mock_update.assert_awaited_once()+ kwargs = mock_update.await_args.kwargs+ positional = mock_update.await_args.args+ assert positional[0] == "wrb_1"+ assert positional[1] == BlockStatus.failed+ assert kwargs["task_status"] == TaskStatus.failed+ assert kwargs["step_status"] == StepStatus.failed+ assert kwargs["failure_reason"] == "Illegit complete, data={'error': '...'}"+++async def test_helper_writes_terminated_for_plain_termination():+ e = ScriptTerminationException("Terminate called: no results found")+ with patch(+ "skyvern.services.script_service._update_workflow_block",+ new_callable=AsyncMock,+ ) as mock_update:+ await _handle_script_termination(+ e,+ "task block",+ workflow_run_block_id="wrb_2",+ task_id="tsk_2",+ step_id="stp_2",+ cache_key="MyTaskBlock",+ )+ mock_update.assert_awaited_once()+ kwargs = mock_update.await_args.kwargs+ positional = mock_update.await_args.args+ assert positional[0] == "wrb_2"+ assert positional[1] == BlockStatus.terminated+ assert kwargs["task_status"] == TaskStatus.terminated+ assert kwargs["step_status"] == StepStatus.failed+ assert kwargs["failure_reason"] == "Terminate called: no results found"+++async def test_helper_skips_db_write_when_no_workflow_run_block_id():+ e = ScriptTerminationException("Terminate called")+ with patch(+ "skyvern.services.script_service._update_workflow_block",+ new_callable=AsyncMock,+ ) as mock_update:+ await _handle_script_termination(+ e,+ "task block",+ workflow_run_block_id=None,+ task_id=None,+ step_id=None,+ cache_key="MyTaskBlock",+ )+ mock_update.assert_not_awaited()diff --git a/tests/unit/test_script_skyvern_page.py b/tests/unit/test_script_skyvern_page.pyindex e71a2ef67..06b1f0b47 100644--- a/tests/unit/test_script_skyvern_page.py+++ b/tests/unit/test_script_skyvern_page.py@@ -16,7 +16,7 @@ import pytestfrom skyvern.config import settingsfrom skyvern.core.script_generations.script_skyvern_page import ScriptSkyvernPage-from skyvern.exceptions import ScriptTerminationException+from skyvern.exceptions import IllegitCompleteScriptTermination, ScriptTerminationExceptiondef create_mock_page():@@ -200,7 +200,8 @@ async def test_wait_for_page_ready_attribute_access_regression():source = inspect.getsource(ScriptSkyvernPage._wait_for_page_ready_before_action)# The fixed code should use self.page- assert "self.page" in source, "Method should access self.page"+ # nosemgrep false positive: "self.page" is an attribute name, not a URL.+ assert "self.page" in source, "Method should access self.page" # nosemgrep: incomplete-url-substring-sanitization# The fixed code should NOT use self._page (except in comments)# Remove comments and docstrings first@@ -560,6 +561,87 @@ async def test_terminate_raises_even_when_handler_fails(mock_scraped_page, mock_mock_handler.assert_called_once()+async def test_complete_raises_illegit_complete_subclass_when_handler_rejects(mock_scraped_page, mock_ai):+ """+ When handle_complete_action returns an ActionFailure (verifier rejected the+ completion), complete() must raise IllegitCompleteScriptTermination — the+ subclass — not the parent ScriptTerminationException. The script_service+ catch sites distinguish the two: subclass → BlockStatus.failed (fallback+ fires), parent → BlockStatus.terminated (no fallback).+ """+ mock_page = create_mock_page()++ with patch(+ "skyvern.core.script_generations.skyvern_page.Page.__init__",+ return_value=None,+ ):+ script_page = ScriptSkyvernPage(+ scraped_page=mock_scraped_page,+ page=mock_page,+ ai=mock_ai,+ )++ mock_context = MagicMock()+ mock_context.organization_id = "org_123"+ mock_context.workflow_run_id = "wr_456"+ mock_context.task_id = "tsk_789"+ mock_context.step_id = "stp_012"+ mock_context.action_order = 0+ mock_context.skip_complete_verification = False+ mock_context.code_version = 1+ mock_context.is_static_script = False+ mock_context.script_mode = False++ mock_task = MagicMock()+ mock_step = MagicMock()+ mock_step.order = 0++ rejected_result = MagicMock(success=False)+ rejected_result.exception_message = (+ "Illegit complete, data={'error': 'Goal not achieved — page still on landing'}"+ )++ with (+ patch(+ "skyvern.core.script_generations.script_skyvern_page.skyvern_context.current",+ return_value=mock_context,+ ),+ patch(+ "skyvern.core.script_generations.script_skyvern_page.app.DATABASE.tasks.get_task",+ new_callable=AsyncMock,+ return_value=mock_task,+ ),+ patch(+ "skyvern.core.script_generations.script_skyvern_page.app.DATABASE.tasks.get_step",+ new_callable=AsyncMock,+ return_value=mock_step,+ ),+ patch.object(+ ScriptSkyvernPage,+ "_update_step_output_before_complete",+ new_callable=AsyncMock,+ ),+ patch.object(+ ScriptSkyvernPage,+ "_create_final_screenshot",+ new_callable=AsyncMock,+ ),+ patch(+ "skyvern.core.script_generations.script_skyvern_page.handle_complete_action",+ new_callable=AsyncMock,+ return_value=[rejected_result],+ ),+ ):+ # The subclass is what the catch sites in script_service.py key off of+ # to record BlockStatus.failed (so AI fallback fires). If complete()+ # regresses to raising plain ScriptTerminationException, the parent+ # arm catches it and the block is recorded as terminated — no+ # fallback fires for what is genuinely a failure.+ with pytest.raises(IllegitCompleteScriptTermination, match="Illegit complete"):+ await script_page.complete()++# =============================================================================# Tests for fill() proactive upgrade when value=None + prompt# =============================================================================