Skip to content

Automate browser based workflows with AI

fix: map ai_click operational failures to 422 on /v1/sdk/run_action (#6065)

Aaron Perez47892c98e250
diff --git a/skyvern/core/script_generations/real_skyvern_page_ai.py b/skyvern/core/script_generations/real_skyvern_page_ai.py
index ac143212f..26a6e5c45 100644
--- a/skyvern/core/script_generations/real_skyvern_page_ai.py
+++ b/skyvern/core/script_generations/real_skyvern_page_ai.py
@@ -12,7 +12,7 @@ from playwright.async_api import Page
from skyvern.config import settings
from skyvern.constants import SKYVERN_PAGE_MAX_SCRAPING_RETRIES, SPECIAL_FIELD_VERIFICATION_CODE
from skyvern.core.script_generations.skyvern_page_ai import SYSTEM_PROMPT_UNSET, SkyvernPageAi
-from skyvern.exceptions import WorkflowRunContextNotInitialized
+from skyvern.exceptions import SkyvernActionFailed, WorkflowRunContextNotInitialized
from skyvern.forge import app
from skyvern.forge.prompts import prompt_engine
from skyvern.forge.sdk.api.files import validate_download_url
@@ -207,9 +207,7 @@ class RealSkyvernPageAi(SkyvernPageAi):
)
actions_json = json_response.get("actions", [])
if not actions_json:
- # LLM returned no actions - the element likely doesn't exist on the page.
- # Raise an exception so the caller knows the action failed.
- raise Exception(
+ raise SkyvernActionFailed(
f"AI could not find an element to click for intention: {intention}. "
"The element may not exist on the current page."
)
@@ -222,7 +220,7 @@ class RealSkyvernPageAi(SkyvernPageAi):
action = cast(ClickAction, actions[0])
result = await handle_click_action(action, self.page, self.scraped_page, task, step)
if result and result[-1].success is False:
- raise Exception(result[-1].exception_message)
+ raise SkyvernActionFailed(result[-1].exception_message or "Click action returned success=False")
xpath = action.get_xpath()
selector = f"xpath={xpath}" if xpath else selector
@@ -241,7 +239,17 @@ class RealSkyvernPageAi(SkyvernPageAi):
)
return selector
+ except SkyvernActionFailed:
+ if selector is None:
+ raise
+ LOG.warning(
+ "AI click failed, falling back to original selector",
+ selector=selector,
+ intention=intention,
+ )
except Exception:
+ if selector is None:
+ raise
LOG.exception(
f"Failed to do ai click. Falling back to original selector={selector}, intention={intention}, data={data}"
)
@@ -251,8 +259,7 @@ class RealSkyvernPageAi(SkyvernPageAi):
await locator.click(timeout=timeout)
return selector
- # If we reach here with no selector, the AI failed and there's no fallback - raise an error
- raise Exception(f"AI click failed and no fallback selector available for intention: {intention}")
+ raise SkyvernActionFailed(f"AI click failed and no fallback selector available for intention: {intention}")
async def ai_input_text(
self,
diff --git a/skyvern/exceptions.py b/skyvern/exceptions.py
index e5366d91b..b3ab55085 100644
--- a/skyvern/exceptions.py
+++ b/skyvern/exceptions.py
@@ -533,6 +533,14 @@ class ScrapingFailed(SkyvernException):
super().__init__("Scraping failed.")
+class SkyvernActionFailed(SkyvernException):
+ """Operationally-expected failure during an SDK action execution."""
+
+ def __init__(self, reason: str) -> None:
+ self.reason = reason
+ super().__init__(reason)
+
+
class ScrapingFailedBlankPage(ScrapingFailed):
def __init__(self) -> None:
super().__init__(reason="It's a blank page. Please ensure there is a non-blank page for Skyvern to work with.")
diff --git a/skyvern/forge/prompts/skyvern/custom-select.j2 b/skyvern/forge/prompts/skyvern/custom-select.j2
index fc7e8a204..d35fdf7fc 100644
--- a/skyvern/forge/prompts/skyvern/custom-select.j2
+++ b/skyvern/forge/prompts/skyvern/custom-select.j2
@@ -2,6 +2,7 @@ You are performing a {{ "multi-level selection" if select_history else "selectio
You can identify the matching element based on the following guidelines:
- Select the most suitable element based on the user goal, user details, and the context.
+ - Match options only by their visible text, labels, or semantic HTML attributes (e.g. aria-label, title). Do NOT derive meaning from element IDs — they are technical identifiers that may be autogenerated (e.g. "ADuM", "ADuS") and do not encode option semantics.
- If none of the options perfectly match, and there is no search box for input, but there is a fallback option such as "Other", "Others" or "None of the above" in the HTML elements, you can consider it a match.
- If a field is required, do not leave it blank.
- If a field is required, do not select a placeholder value, such as "Please select", "-", or "Select...".
diff --git a/skyvern/forge/sdk/routes/sdk.py b/skyvern/forge/sdk/routes/sdk.py
index be1081a79..cc30e43d7 100644
--- a/skyvern/forge/sdk/routes/sdk.py
+++ b/skyvern/forge/sdk/routes/sdk.py
@@ -5,7 +5,7 @@ from fastapi import Depends, HTTPException, status
from skyvern.core.script_generations.real_skyvern_page_ai import RealSkyvernPageAi
from skyvern.core.script_generations.script_skyvern_page import ScriptSkyvernPage
-from skyvern.exceptions import ScrapingFailed
+from skyvern.exceptions import ScrapingFailed, SkyvernActionFailed
from skyvern.forge import app
from skyvern.forge.sdk.api.files import validate_download_url
from skyvern.forge.sdk.core import skyvern_context
@@ -242,6 +242,19 @@ async def run_sdk_action(
error=str(e),
)
raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=e.reason or str(e))
+ except SkyvernActionFailed as e:
+ await app.DATABASE.tasks.update_task(
+ task_id=task.task_id,
+ organization_id=organization_id,
+ status=TaskStatus.failed,
+ failure_reason=e.reason,
+ )
+ LOG.warning(
+ "SDK action failed",
+ action_type=action.type,
+ error=e.reason,
+ )
+ raise HTTPException(status_code=status.HTTP_422_UNPROCESSABLE_CONTENT, detail=e.reason)
except Exception as e:
await app.DATABASE.tasks.update_task(
task_id=task.task_id,
diff --git a/tests/unit/test_ai_click_empty_actions.py b/tests/unit/test_ai_click_empty_actions.py
index bed5bc786..0307b0741 100644
--- a/tests/unit/test_ai_click_empty_actions.py
+++ b/tests/unit/test_ai_click_empty_actions.py
@@ -12,6 +12,7 @@ from unittest.mock import AsyncMock, MagicMock, patch
import pytest
from skyvern.core.script_generations.real_skyvern_page_ai import RealSkyvernPageAi
+from skyvern.exceptions import SkyvernActionFailed
@pytest.fixture
@@ -84,26 +85,26 @@ class TestAiClickEmptyActions:
return_value="mock_prompt",
),
):
- with pytest.raises(Exception) as exc_info:
+ with pytest.raises(SkyvernActionFailed) as exc_info:
await real_skyvern_page_ai.ai_click(
- selector=None, # No fallback selector
+ selector=None,
intention="Click the download button",
)
- # Should raise because no actions and no fallback
assert "AI click failed" in str(exc_info.value) or "AI could not find" in str(exc_info.value)
@pytest.mark.asyncio
- async def test_ai_click_raises_when_llm_call_fails_no_selector(
+ async def test_ai_click_propagates_unknown_exception_when_no_selector(
self, mock_page, mock_scraped_page, mock_context, mock_app
):
"""
- When AI fails (exception) and there's no selector to fall back to,
- ai_click should raise an exception.
+ When the AI path raises a non-operational exception (LLM down, etc.) and
+ there's no selector to fall back to, the original exception should
+ propagate so the caller surfaces it as a 500, not a 422.
"""
real_skyvern_page_ai = RealSkyvernPageAi(mock_scraped_page, mock_page)
- mock_app.SINGLE_CLICK_AGENT_LLM_API_HANDLER = AsyncMock(side_effect=Exception("LLM error"))
+ mock_app.SINGLE_CLICK_AGENT_LLM_API_HANDLER = AsyncMock(side_effect=RuntimeError("LLM provider down"))
with (
patch.object(real_skyvern_page_ai, "_refresh_scraped_page", new_callable=AsyncMock),
@@ -117,13 +118,14 @@ class TestAiClickEmptyActions:
return_value="mock_prompt",
),
):
- with pytest.raises(Exception) as exc_info:
+ with pytest.raises(RuntimeError) as exc_info:
await real_skyvern_page_ai.ai_click(
- selector=None, # No fallback selector
+ selector=None,
intention="Click the download button",
)
- assert "AI click failed" in str(exc_info.value)
+ assert "LLM provider down" in str(exc_info.value)
+ assert not isinstance(exc_info.value, SkyvernActionFailed)
@pytest.mark.asyncio
async def test_ai_click_falls_back_to_selector_when_llm_returns_empty(
diff --git a/tests/unit/test_run_sdk_action_failures.py b/tests/unit/test_run_sdk_action_failures.py
new file mode 100644
index 000000000..1d87f2338
--- /dev/null
+++ b/tests/unit/test_run_sdk_action_failures.py
@@ -0,0 +1,112 @@
+from __future__ import annotations
+
+from typing import Any
+from unittest.mock import AsyncMock, MagicMock, patch
+
+import pytest
+from fastapi import HTTPException
+
+from skyvern.exceptions import ScrapingFailed, SkyvernActionFailed
+from skyvern.forge.sdk.routes.sdk import run_sdk_action
+
+
+def mock_request() -> Any:
+ request = MagicMock()
+ request.workflow_run_id = "wr_test"
+ request.browser_session_id = None
+ request.browser_address = None
+ request.url = "https://example.com"
+ request.action = MagicMock()
+ request.action.type = "ai_click"
+ request.action.selector = None
+ request.action.intention = "Click the button"
+ request.action.data = None
+ request.action.timeout = 30000
+ request.action.get_navigation_goal = MagicMock(return_value="Click the button")
+ request.action.get_navigation_payload = MagicMock(return_value=None)
+ return request
+
+
+def mock_organization() -> Any:
+ org = MagicMock()
+ org.organization_id = "o_test"
+ return org
+
+
+def mock_app() -> Any:
+ app = MagicMock()
+ workflow_run = MagicMock(workflow_run_id="wr_test", workflow_id="w_test")
+ workflow = MagicMock(workflow_id="w_test", workflow_permanent_id="wpid_test", title="t")
+ task = MagicMock(task_id="tsk_test", organization_id="o_test", max_screenshot_scrolls=None)
+ step = MagicMock(step_id="stp_test")
+ app.DATABASE.workflow_runs.get_workflow_run = AsyncMock(return_value=workflow_run)
+ app.DATABASE.workflows.get_workflow = AsyncMock(return_value=workflow)
+ app.DATABASE.tasks.create_task = AsyncMock(return_value=task)
+ app.DATABASE.tasks.create_step = AsyncMock(return_value=step)
+ app.DATABASE.tasks.update_task = AsyncMock()
+ app.DATABASE.observer.create_workflow_run_block = AsyncMock()
+ app.WORKFLOW_CONTEXT_MANAGER.initialize_workflow_run_context = AsyncMock()
+ return app
+
+
+async def test_handler_returns_422_when_action_raises_skyvern_action_failed(
+ mock_request: Any, mock_organization: Any, mock_app: Any
+) -> None:
+ with (
+ patch("skyvern.forge.sdk.routes.sdk.app", mock_app),
+ patch("skyvern.forge.sdk.routes.sdk.skyvern_context") as mock_ctx,
+ patch(
+ "skyvern.core.script_generations.script_skyvern_page.ScriptSkyvernPage.create_scraped_page",
+ new_callable=AsyncMock,
+ side_effect=SkyvernActionFailed("AI click failed and no fallback selector available"),
+ ),
+ ):
+ mock_ctx.ensure_context.return_value = MagicMock(request_id="req_test", tz_info=None, prompt=None)
+ with pytest.raises(HTTPException) as exc_info:
+ await run_sdk_action(mock_request, organization=mock_organization)
+
+ assert exc_info.value.status_code == 422
+ assert "AI click failed" in str(exc_info.value.detail)
+ mock_app.DATABASE.tasks.update_task.assert_awaited()
+
+
+async def test_handler_returns_400_when_action_raises_scraping_failed(
+ mock_request: Any, mock_organization: Any, mock_app: Any
+) -> None:
+ with (
+ patch("skyvern.forge.sdk.routes.sdk.app", mock_app),
+ patch("skyvern.forge.sdk.routes.sdk.skyvern_context") as mock_ctx,
+ patch(
+ "skyvern.core.script_generations.script_skyvern_page.ScriptSkyvernPage.create_scraped_page",
+ new_callable=AsyncMock,
+ side_effect=ScrapingFailed(reason="page is blank"),
+ ),
+ ):
+ mock_ctx.ensure_context.return_value = MagicMock(request_id="req_test", tz_info=None, prompt=None)
+ with pytest.raises(HTTPException) as exc_info:
+ await run_sdk_action(mock_request, organization=mock_organization)
+
+ assert exc_info.value.status_code == 400
+
+
+async def test_handler_propagates_unknown_exception(mock_request: Any, mock_organization: Any, mock_app: Any) -> None:
+ with (
+ patch("skyvern.forge.sdk.routes.sdk.app", mock_app),
+ patch("skyvern.forge.sdk.routes.sdk.skyvern_context") as mock_ctx,
+ patch(
+ "skyvern.core.script_generations.script_skyvern_page.ScriptSkyvernPage.create_scraped_page",
+ new_callable=AsyncMock,
+ side_effect=RuntimeError("db connection pool exhausted"),
+ ),
+ ):
+ mock_ctx.ensure_context.return_value = MagicMock(request_id="req_test", tz_info=None, prompt=None)
+ with pytest.raises(RuntimeError):
+ await run_sdk_action(mock_request, organization=mock_organization)
+
+ mock_app.DATABASE.tasks.update_task.assert_awaited()