Skip to content

Automate browser based workflows with AI

fix(SKY-9568): persist block status on script termination, distinguish complete-reject from intentional terminate (#5862)

pedrohsdbdddbe89fb754
diff --git a/skyvern/core/script_generations/script_skyvern_page.py b/skyvern/core/script_generations/script_skyvern_page.py
index 47418bb4d..a2d4c8059 100644
--- a/skyvern/core/script_generations/script_skyvern_page.py
+++ b/skyvern/core/script_generations/script_skyvern_page.py
@@ -19,7 +19,7 @@ from skyvern.core.script_generations.real_skyvern_page_ai import RealSkyvernPage
from skyvern.core.script_generations.skyvern_page import ActionCall, ActionMetadata, RunContext, SkyvernPage
from skyvern.core.script_generations.skyvern_page_ai import SkyvernPageAi
from skyvern.errors.errors import UserDefinedError
-from skyvern.exceptions import ScriptTerminationException, WorkflowRunNotFound
+from skyvern.exceptions import IllegitCompleteScriptTermination, ScriptTerminationException, WorkflowRunNotFound
from skyvern.forge import app
from skyvern.forge.prompts import prompt_engine
from skyvern.forge.sdk.api.files import (
@@ -1076,7 +1076,9 @@ class ScriptSkyvernPage(SkyvernPage):
# result = await ActionHandler.handle_action(self.scraped_page, task, step, self.page, action)
result = await handle_complete_action(action, self.page, self.scraped_page, task, step)
if result and result[-1].success is False:
- raise ScriptTerminationException(result[-1].exception_message)
+ # Coerce empty/None messages so downstream str(e) is meaningful.
+ msg = result[-1].exception_message or "complete-verify rejected without a message"
+ raise IllegitCompleteScriptTermination(msg)
# Capture final full-page screenshot at block completion
await self._create_final_screenshot()
diff --git a/skyvern/exceptions.py b/skyvern/exceptions.py
index f39cfc64a..244fead42 100644
--- a/skyvern/exceptions.py
+++ b/skyvern/exceptions.py
@@ -1143,6 +1143,10 @@ class ScriptTerminationException(SkyvernException):
super().__init__(reason)
+class IllegitCompleteScriptTermination(ScriptTerminationException):
+ """Raised when a cached script's page.complete() is rejected by the verifier; distinct from plain ScriptTerminationException, which is an intentional terminate()."""
+
+
class InvalidSchemaError(SkyvernException):
def __init__(self, message: str, validation_errors: list[str] | None = None):
self.message = message
diff --git a/skyvern/forge/sdk/workflow/service.py b/skyvern/forge/sdk/workflow/service.py
index 402510249..d3c181510 100644
--- a/skyvern/forge/sdk/workflow/service.py
+++ b/skyvern/forge/sdk/workflow/service.py
@@ -2406,6 +2406,10 @@ class WorkflowService:
LOG.debug("Executing run_signature wrapper", wrapper_code=wrapper_code)
+ # Pre-init so the success log can reference output_value
+ # when ScriptTerminationException was caught (terminated
+ # is now in script_success_statuses).
+ output_value: Any = None
try:
exec_code = compile(wrapper_code, "<run_signature>", "exec")
exec(exec_code, exec_globals)
@@ -2433,10 +2437,10 @@ class WorkflowService:
status=BlockStatus(latest_block.status) if latest_block.status else BlockStatus.failed,
workflow_run_block_id=latest_block.workflow_run_block_id,
)
- # Terminated is a valid script outcome when generate_script_on_terminal is set
- script_success_statuses = {BlockStatus.completed}
- if workflow.generate_script_on_terminal:
- script_success_statuses.add(BlockStatus.terminated)
+ # Terminate() is an explicit signal to stop, not a
+ # failure to retry. `generate_script_on_terminal` is
+ # orthogonal — it gates script generation, not fallback.
+ script_success_statuses = {BlockStatus.completed, BlockStatus.terminated}
block_exec_duration_ms = round((time.monotonic() - block_exec_start) * 1000, 1)
if workflow_run_block_result.status in script_success_statuses:
@@ -2603,7 +2607,36 @@ class WorkflowService:
# if the failure is code-fixable.
if fallback_episode_id and workflow_run_block_result:
try:
+ # None = unknown count (taskless wrb / fetch error);
+ # only a confirmed zero downgrades fallback_succeeded.
+ fallback_wrb_id = workflow_run_block_result.workflow_run_block_id
+ agent_action_count: int | None = None
+ action_summaries: list[dict] | None = None
+ if fallback_wrb_id:
+ try:
+ wrb = await app.DATABASE.observer.get_workflow_run_block(
+ workflow_run_block_id=fallback_wrb_id,
+ organization_id=organization_id,
+ )
+ if wrb and wrb.task_id:
+ actions = await app.DATABASE.tasks.get_task_actions(
+ task_id=wrb.task_id,
+ organization_id=organization_id,
+ )
+ agent_action_count = len(actions)
+ action_summaries = build_action_summaries_with_timing(actions)
+ except Exception:
+ LOG.debug(
+ "Could not fetch rich actions for fallback episode",
+ fallback_wrb_id=fallback_wrb_id,
+ exc_info=True,
+ )
+
+ # `completed` with confirmed 0 actions = the agent's
+ # complete-verify accepting what the script's rejected.
fallback_succeeded = workflow_run_block_result.status == BlockStatus.completed
+ if fallback_succeeded and agent_action_count == 0:
+ fallback_succeeded = False
# Build agent actions summary for both success and failure
agent_actions_summary: dict = {
@@ -2614,42 +2647,24 @@ class WorkflowService:
}
if form_fields_for_episode:
agent_actions_summary["form_fields"] = form_fields_for_episode
+ if action_summaries is not None:
+ agent_actions_summary["actions"] = action_summaries
- # For failed fallbacks, capture the failure reason
if not fallback_succeeded:
- agent_actions_summary["failure_reason"] = (
- str(workflow_run_block_result.failure_reason)[:2000]
- if workflow_run_block_result.failure_reason
- else None
- )
+ if workflow_run_block_result.failure_reason:
+ agent_actions_summary["failure_reason"] = str(workflow_run_block_result.failure_reason)[
+ :2000
+ ]
+ elif workflow_run_block_result.status == BlockStatus.completed and agent_action_count == 0:
+ agent_actions_summary["failure_reason"] = script_service.VERIFIER_SWAP_FAILURE_REASON
LOG.info(
"AI fallback failed, keeping episode for triage",
episode_id=fallback_episode_id,
block_status=workflow_run_block_result.status,
block_label=block.label,
+ agent_action_count=agent_action_count,
)
- # Fetch rich action details from the fallback execution
- fallback_wrb_id = workflow_run_block_result.workflow_run_block_id
- if fallback_wrb_id:
- try:
- wrb = await app.DATABASE.observer.get_workflow_run_block(
- workflow_run_block_id=fallback_wrb_id,
- organization_id=organization_id,
- )
- if wrb and wrb.task_id:
- actions = await app.DATABASE.tasks.get_task_actions(
- task_id=wrb.task_id,
- organization_id=organization_id,
- )
- agent_actions_summary["actions"] = build_action_summaries_with_timing(actions)
- except Exception:
- LOG.debug(
- "Could not fetch rich actions for fallback episode",
- fallback_wrb_id=fallback_wrb_id,
- exc_info=True,
- )
-
await app.DATABASE.scripts.update_fallback_episode(
episode_id=fallback_episode_id,
organization_id=organization_id,
diff --git a/skyvern/services/script_service.py b/skyvern/services/script_service.py
index 8579f994d..a260793a3 100644
--- a/skyvern/services/script_service.py
+++ b/skyvern/services/script_service.py
@@ -29,6 +29,7 @@ from skyvern.core.script_generations.script_skyvern_page import script_run_conte
from skyvern.errors.errors import UserDefinedError, filter_to_user_defined_codes
from skyvern.exceptions import (
CachedDownloadError,
+ IllegitCompleteScriptTermination,
ScriptNotFound,
ScriptTerminationException,
StepTerminationError,
@@ -91,6 +92,16 @@ from skyvern.webeye.scraper.scraped_page import ElementTreeFormat
LOG = structlog.get_logger()
jinja_sandbox_env = SandboxedEnvironment()
+# Synthetic failure_reason recorded on a fallback episode when the AI fallback
+# ended `completed` with zero actions taken — i.e. the AI's complete-verify
+# accepted what the script's complete-verify rejected, so the agent didn't
+# actually do anything. Shared between the two episode writers
+# (script_service._fallback_to_ai_run + workflow/service.py:_execute_single_block)
+# so they stay in sync.
+VERIFIER_SWAP_FAILURE_REASON = (
+ "AI fallback ended completed with 0 actions — complete-verify accepted what the script's complete-verify rejected"
+)
+
# Max wait for any download signal after a cached click; downstream
# .crdownload polling handles in-progress completion separately. (SKY-9431)
CACHED_DOWNLOAD_NO_FILE_GRACE_SECONDS = 60
@@ -623,6 +634,43 @@ async def _record_output_parameter_value(
)
+async def _handle_script_termination(
+ e: ScriptTerminationException,
+ block_kind: str,
+ workflow_run_block_id: str | None,
+ task_id: str | None,
+ step_id: str | None,
+ cache_key: str,
+) -> None:
+ """Persist the block as failed (verifier-rejected complete) or terminated (intentional terminate).
+
+ StepStatus uses `failed` for both: the enum has no `terminated` value (only created/running/failed/completed/canceled), and `failed` is the existing codebase convention for non-completed steps.
+ """
+ if isinstance(e, IllegitCompleteScriptTermination):
+ LOG.info("Script complete() rejected by verifier", block_kind=block_kind, cache_key=cache_key)
+ block_status = BlockStatus.failed
+ task_status = TaskStatus.failed
+ else:
+ LOG.info(
+ "Script requested termination, not falling back to AI",
+ block_kind=block_kind,
+ cache_key=cache_key,
+ )
+ block_status = BlockStatus.terminated
+ task_status = TaskStatus.terminated
+ if workflow_run_block_id:
+ await _update_workflow_block(
+ workflow_run_block_id,
+ block_status,
+ task_id=task_id,
+ task_status=task_status,
+ step_id=step_id,
+ step_status=StepStatus.failed,
+ label=cache_key,
+ failure_reason=str(e),
+ )
+
+
async def _update_workflow_block(
workflow_run_block_id: str,
status: BlockStatus,
@@ -1402,23 +1450,42 @@ async def _fallback_to_ai_run(
# Update fallback episode with AI execution results
if fallback_episode_id:
try:
- fallback_succeeded = task.status not in [TaskStatus.terminated, TaskStatus.failed]
- agent_actions_summary: dict[str, Any] = {
- "block_status": str(task.status),
- }
- if form_fields_snapshot:
- agent_actions_summary["form_fields"] = form_fields_snapshot
- if not fallback_succeeded and task.failure_reason:
- agent_actions_summary["failure_reason"] = str(task.failure_reason)[:2000]
+ # AI runs on the existing task and may retry across steps
+ # (agent.execute_step recurses on next_step). Exclude the
+ # script's pre-fallback step only — retry-chain actions are
+ # counted. None = unknown (fetch error).
+ agent_action_count: int | None = None
+ action_summaries: list[dict] | None = None
try:
- actions = await app.DATABASE.tasks.get_task_actions(
+ all_actions = await app.DATABASE.tasks.get_task_actions(
task_id=task_id,
organization_id=organization_id,
)
- agent_actions_summary["actions"] = build_action_summaries_with_timing(actions)
+ ai_actions = [a for a in all_actions if a.step_id != script_step_id]
+ agent_action_count = len(ai_actions)
+ action_summaries = build_action_summaries_with_timing(ai_actions)
except Exception:
LOG.debug("Could not fetch actions for fallback episode", exc_info=True)
+ # Mirrors workflow/service.py's success predicate so the two
+ # episode writers label the same outcome identically.
+ fallback_succeeded = task.status == TaskStatus.completed
+ if fallback_succeeded and agent_action_count == 0:
+ fallback_succeeded = False
+
+ agent_actions_summary: dict[str, Any] = {
+ "block_status": str(task.status),
+ }
+ if form_fields_snapshot:
+ agent_actions_summary["form_fields"] = form_fields_snapshot
+ if action_summaries is not None:
+ agent_actions_summary["actions"] = action_summaries
+ if not fallback_succeeded:
+ if task.failure_reason:
+ agent_actions_summary["failure_reason"] = str(task.failure_reason)[:2000]
+ elif task.status == TaskStatus.completed and agent_action_count == 0:
+ agent_actions_summary["failure_reason"] = VERIFIER_SWAP_FAILURE_REASON
+
await app.DATABASE.scripts.update_fallback_episode(
episode_id=fallback_episode_id,
organization_id=organization_id,
@@ -1809,10 +1876,8 @@ async def run_task(
)
return output
- except ScriptTerminationException:
- # Explicit termination from script (e.g., target data does not exist).
- # Do NOT fall back to AI — the script intentionally terminated.
- LOG.info("Script requested termination. Not falling back to AI.", cache_key=cache_key)
+ except ScriptTerminationException as e:
+ await _handle_script_termination(e, "task block", workflow_run_block_id, task_id, step_id, cache_key)
raise
except Exception as e:
LOG.exception("Failed to run task block. Falling back to AI run.")
@@ -2093,10 +2158,8 @@ async def download(
label=cache_key,
)
- except ScriptTerminationException:
- # Explicit termination from script (e.g., target data does not exist).
- # Do NOT fall back to AI — the script intentionally terminated.
- LOG.info("Script requested termination in download block. Not falling back to AI.", cache_key=cache_key)
+ except ScriptTerminationException as e:
+ await _handle_script_termination(e, "download block", workflow_run_block_id, task_id, step_id, cache_key)
raise
except Exception as e:
LOG.exception("Failed to run download block. Falling back to AI run.")
@@ -2185,10 +2248,8 @@ async def action(
label=cache_key,
)
- except ScriptTerminationException:
- # Explicit termination from script (e.g., target data does not exist).
- # Do NOT fall back to AI — the script intentionally terminated.
- LOG.info("Script requested termination in action block. Not falling back to AI.", cache_key=cache_key)
+ except ScriptTerminationException as e:
+ await _handle_script_termination(e, "action block", workflow_run_block_id, task_id, step_id, cache_key)
raise
except Exception as e:
LOG.exception("Failed to run action block. Falling back to AI run.")
@@ -2280,10 +2341,8 @@ async def login(
label=cache_key,
)
- except ScriptTerminationException:
- # Explicit termination from script (e.g., target data does not exist).
- # Do NOT fall back to AI — the script intentionally terminated.
- LOG.info("Script requested termination in login block. Not falling back to AI.", cache_key=cache_key)
+ except ScriptTerminationException as e:
+ await _handle_script_termination(e, "login block", workflow_run_block_id, task_id, step_id, cache_key)
raise
except Exception as e:
LOG.exception("Failed to run login block")
diff --git a/tests/unit/test_handle_script_termination.py b/tests/unit/test_handle_script_termination.py
new file mode 100644
index 000000000..cbe03bfbc
--- /dev/null
+++ b/tests/unit/test_handle_script_termination.py
@@ -0,0 +1,84 @@
+"""Locks the bookkeeping contract for _handle_script_termination (SKY-9568):
+IllegitCompleteScriptTermination -> BlockStatus.failed (so AI fallback fires);
+plain ScriptTerminationException -> BlockStatus.terminated (no fallback)."""
+
+from unittest.mock import AsyncMock, patch
+
+import pytest
+
+from skyvern.exceptions import (
+ IllegitCompleteScriptTermination,
+ ScriptTerminationException,
+)
+from skyvern.forge.sdk.models import StepStatus
+from skyvern.forge.sdk.schemas.tasks import TaskStatus
+from skyvern.schemas.workflows import BlockStatus
+from skyvern.services.script_service import _handle_script_termination
+
+
+async def test_helper_writes_failed_for_illegit_complete():
+ e = IllegitCompleteScriptTermination("Illegit complete, data={'error': '...'}")
+ with patch(
+ "skyvern.services.script_service._update_workflow_block",
+ new_callable=AsyncMock,
+ ) as mock_update:
+ await _handle_script_termination(
+ e,
+ "task block",
+ workflow_run_block_id="wrb_1",
+ task_id="tsk_1",
+ step_id="stp_1",
+ cache_key="MyTaskBlock",
+ )
+ mock_update.assert_awaited_once()
+ kwargs = mock_update.await_args.kwargs
+ positional = mock_update.await_args.args
+ assert positional[0] == "wrb_1"
+ assert positional[1] == BlockStatus.failed
+ assert kwargs["task_status"] == TaskStatus.failed
+ assert kwargs["step_status"] == StepStatus.failed
+ assert kwargs["failure_reason"] == "Illegit complete, data={'error': '...'}"
+
+
+async def test_helper_writes_terminated_for_plain_termination():
+ e = ScriptTerminationException("Terminate called: no results found")
+ with patch(
+ "skyvern.services.script_service._update_workflow_block",
+ new_callable=AsyncMock,
+ ) as mock_update:
+ await _handle_script_termination(
+ e,
+ "task block",
+ workflow_run_block_id="wrb_2",
+ task_id="tsk_2",
+ step_id="stp_2",
+ cache_key="MyTaskBlock",
+ )
+ mock_update.assert_awaited_once()
+ kwargs = mock_update.await_args.kwargs
+ positional = mock_update.await_args.args
+ assert positional[0] == "wrb_2"
+ assert positional[1] == BlockStatus.terminated
+ assert kwargs["task_status"] == TaskStatus.terminated
+ assert kwargs["step_status"] == StepStatus.failed
+ assert kwargs["failure_reason"] == "Terminate called: no results found"
+
+
+async def test_helper_skips_db_write_when_no_workflow_run_block_id():
+ e = ScriptTerminationException("Terminate called")
+ with patch(
+ "skyvern.services.script_service._update_workflow_block",
+ new_callable=AsyncMock,
+ ) as mock_update:
+ await _handle_script_termination(
+ e,
+ "task block",
+ workflow_run_block_id=None,
+ task_id=None,
+ step_id=None,
+ cache_key="MyTaskBlock",
+ )
+ mock_update.assert_not_awaited()
diff --git a/tests/unit/test_script_skyvern_page.py b/tests/unit/test_script_skyvern_page.py
index e71a2ef67..06b1f0b47 100644
--- a/tests/unit/test_script_skyvern_page.py
+++ b/tests/unit/test_script_skyvern_page.py
@@ -16,7 +16,7 @@ import pytest
from skyvern.config import settings
from skyvern.core.script_generations.script_skyvern_page import ScriptSkyvernPage
-from skyvern.exceptions import ScriptTerminationException
+from skyvern.exceptions import IllegitCompleteScriptTermination, ScriptTerminationException
def create_mock_page():
@@ -200,7 +200,8 @@ async def test_wait_for_page_ready_attribute_access_regression():
source = inspect.getsource(ScriptSkyvernPage._wait_for_page_ready_before_action)
# The fixed code should use self.page
- assert "self.page" in source, "Method should access self.page"
+ # nosemgrep false positive: "self.page" is an attribute name, not a URL.
+ assert "self.page" in source, "Method should access self.page" # nosemgrep: incomplete-url-substring-sanitization
# The fixed code should NOT use self._page (except in comments)
# Remove comments and docstrings first
@@ -560,6 +561,87 @@ async def test_terminate_raises_even_when_handler_fails(mock_scraped_page, mock_
mock_handler.assert_called_once()
+async def test_complete_raises_illegit_complete_subclass_when_handler_rejects(mock_scraped_page, mock_ai):
+ """
+ When handle_complete_action returns an ActionFailure (verifier rejected the
+ completion), complete() must raise IllegitCompleteScriptTermination — the
+ subclass — not the parent ScriptTerminationException. The script_service
+ catch sites distinguish the two: subclass → BlockStatus.failed (fallback
+ fires), parent → BlockStatus.terminated (no fallback).
+ """
+ mock_page = create_mock_page()
+
+ with patch(
+ "skyvern.core.script_generations.skyvern_page.Page.__init__",
+ return_value=None,
+ ):
+ script_page = ScriptSkyvernPage(
+ scraped_page=mock_scraped_page,
+ page=mock_page,
+ ai=mock_ai,
+ )
+
+ mock_context = MagicMock()
+ mock_context.organization_id = "org_123"
+ mock_context.workflow_run_id = "wr_456"
+ mock_context.task_id = "tsk_789"
+ mock_context.step_id = "stp_012"
+ mock_context.action_order = 0
+ mock_context.skip_complete_verification = False
+ mock_context.code_version = 1
+ mock_context.is_static_script = False
+ mock_context.script_mode = False
+
+ mock_task = MagicMock()
+ mock_step = MagicMock()
+ mock_step.order = 0
+
+ rejected_result = MagicMock(success=False)
+ rejected_result.exception_message = (
+ "Illegit complete, data={'error': 'Goal not achieved — page still on landing'}"
+ )
+
+ with (
+ patch(
+ "skyvern.core.script_generations.script_skyvern_page.skyvern_context.current",
+ return_value=mock_context,
+ ),
+ patch(
+ "skyvern.core.script_generations.script_skyvern_page.app.DATABASE.tasks.get_task",
+ new_callable=AsyncMock,
+ return_value=mock_task,
+ ),
+ patch(
+ "skyvern.core.script_generations.script_skyvern_page.app.DATABASE.tasks.get_step",
+ new_callable=AsyncMock,
+ return_value=mock_step,
+ ),
+ patch.object(
+ ScriptSkyvernPage,
+ "_update_step_output_before_complete",
+ new_callable=AsyncMock,
+ ),
+ patch.object(
+ ScriptSkyvernPage,
+ "_create_final_screenshot",
+ new_callable=AsyncMock,
+ ),
+ patch(
+ "skyvern.core.script_generations.script_skyvern_page.handle_complete_action",
+ new_callable=AsyncMock,
+ return_value=[rejected_result],
+ ),
+ ):
+ # The subclass is what the catch sites in script_service.py key off of
+ # to record BlockStatus.failed (so AI fallback fires). If complete()
+ # regresses to raising plain ScriptTerminationException, the parent
+ # arm catches it and the block is recorded as terminated — no
+ # fallback fires for what is genuinely a failure.
+ with pytest.raises(IllegitCompleteScriptTermination, match="Illegit complete"):
+ await script_page.complete()
+
+
# =============================================================================
# Tests for fill() proactive upgrade when value=None + prompt
# =============================================================================