Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
fix: map ai_click operational failures to 422 on /v1/sdk/run_action (#6065)
Aaron Perez47892c98e250
diff --git a/skyvern/core/script_generations/real_skyvern_page_ai.py b/skyvern/core/script_generations/real_skyvern_page_ai.pyindex ac143212f..26a6e5c45 100644--- a/skyvern/core/script_generations/real_skyvern_page_ai.py+++ b/skyvern/core/script_generations/real_skyvern_page_ai.py@@ -12,7 +12,7 @@ from playwright.async_api import Pagefrom skyvern.config import settingsfrom skyvern.constants import SKYVERN_PAGE_MAX_SCRAPING_RETRIES, SPECIAL_FIELD_VERIFICATION_CODEfrom skyvern.core.script_generations.skyvern_page_ai import SYSTEM_PROMPT_UNSET, SkyvernPageAi-from skyvern.exceptions import WorkflowRunContextNotInitialized+from skyvern.exceptions import SkyvernActionFailed, WorkflowRunContextNotInitializedfrom skyvern.forge import appfrom skyvern.forge.prompts import prompt_enginefrom skyvern.forge.sdk.api.files import validate_download_url@@ -207,9 +207,7 @@ class RealSkyvernPageAi(SkyvernPageAi):)actions_json = json_response.get("actions", [])if not actions_json:- # LLM returned no actions - the element likely doesn't exist on the page.- # Raise an exception so the caller knows the action failed.- raise Exception(+ raise SkyvernActionFailed(f"AI could not find an element to click for intention: {intention}. ""The element may not exist on the current page.")@@ -222,7 +220,7 @@ class RealSkyvernPageAi(SkyvernPageAi):action = cast(ClickAction, actions[0])result = await handle_click_action(action, self.page, self.scraped_page, task, step)if result and result[-1].success is False:- raise Exception(result[-1].exception_message)+ raise SkyvernActionFailed(result[-1].exception_message or "Click action returned success=False")xpath = action.get_xpath()selector = f"xpath={xpath}" if xpath else selector@@ -241,7 +239,17 @@ class RealSkyvernPageAi(SkyvernPageAi):)return selector+ except SkyvernActionFailed:+ if selector is None:+ raise+ LOG.warning(+ "AI click failed, falling back to original selector",+ selector=selector,+ intention=intention,+ )except Exception:+ if selector is None:+ raiseLOG.exception(f"Failed to do ai click. Falling back to original selector={selector}, intention={intention}, data={data}")@@ -251,8 +259,7 @@ class RealSkyvernPageAi(SkyvernPageAi):await locator.click(timeout=timeout)return selector- # If we reach here with no selector, the AI failed and there's no fallback - raise an error- raise Exception(f"AI click failed and no fallback selector available for intention: {intention}")+ raise SkyvernActionFailed(f"AI click failed and no fallback selector available for intention: {intention}")async def ai_input_text(self,diff --git a/skyvern/exceptions.py b/skyvern/exceptions.pyindex e5366d91b..b3ab55085 100644--- a/skyvern/exceptions.py+++ b/skyvern/exceptions.py@@ -533,6 +533,14 @@ class ScrapingFailed(SkyvernException):super().__init__("Scraping failed.")+class SkyvernActionFailed(SkyvernException):+ """Operationally-expected failure during an SDK action execution."""++ def __init__(self, reason: str) -> None:+ self.reason = reason+ super().__init__(reason)++class ScrapingFailedBlankPage(ScrapingFailed):def __init__(self) -> None:super().__init__(reason="It's a blank page. Please ensure there is a non-blank page for Skyvern to work with.")diff --git a/skyvern/forge/prompts/skyvern/custom-select.j2 b/skyvern/forge/prompts/skyvern/custom-select.j2index fc7e8a204..d35fdf7fc 100644--- a/skyvern/forge/prompts/skyvern/custom-select.j2+++ b/skyvern/forge/prompts/skyvern/custom-select.j2@@ -2,6 +2,7 @@ You are performing a {{ "multi-level selection" if select_history else "selectioYou can identify the matching element based on the following guidelines:- Select the most suitable element based on the user goal, user details, and the context.+ - Match options only by their visible text, labels, or semantic HTML attributes (e.g. aria-label, title). Do NOT derive meaning from element IDs — they are technical identifiers that may be autogenerated (e.g. "ADuM", "ADuS") and do not encode option semantics.- If none of the options perfectly match, and there is no search box for input, but there is a fallback option such as "Other", "Others" or "None of the above" in the HTML elements, you can consider it a match.- If a field is required, do not leave it blank.- If a field is required, do not select a placeholder value, such as "Please select", "-", or "Select...".diff --git a/skyvern/forge/sdk/routes/sdk.py b/skyvern/forge/sdk/routes/sdk.pyindex be1081a79..cc30e43d7 100644--- a/skyvern/forge/sdk/routes/sdk.py+++ b/skyvern/forge/sdk/routes/sdk.py@@ -5,7 +5,7 @@ from fastapi import Depends, HTTPException, statusfrom skyvern.core.script_generations.real_skyvern_page_ai import RealSkyvernPageAifrom skyvern.core.script_generations.script_skyvern_page import ScriptSkyvernPage-from skyvern.exceptions import ScrapingFailed+from skyvern.exceptions import ScrapingFailed, SkyvernActionFailedfrom skyvern.forge import appfrom skyvern.forge.sdk.api.files import validate_download_urlfrom skyvern.forge.sdk.core import skyvern_context@@ -242,6 +242,19 @@ async def run_sdk_action(error=str(e),)raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=e.reason or str(e))+ except SkyvernActionFailed as e:+ await app.DATABASE.tasks.update_task(+ task_id=task.task_id,+ organization_id=organization_id,+ status=TaskStatus.failed,+ failure_reason=e.reason,+ )+ LOG.warning(+ "SDK action failed",+ action_type=action.type,+ error=e.reason,+ )+ raise HTTPException(status_code=status.HTTP_422_UNPROCESSABLE_CONTENT, detail=e.reason)except Exception as e:await app.DATABASE.tasks.update_task(task_id=task.task_id,diff --git a/tests/unit/test_ai_click_empty_actions.py b/tests/unit/test_ai_click_empty_actions.pyindex bed5bc786..0307b0741 100644--- a/tests/unit/test_ai_click_empty_actions.py+++ b/tests/unit/test_ai_click_empty_actions.py@@ -12,6 +12,7 @@ from unittest.mock import AsyncMock, MagicMock, patchimport pytestfrom skyvern.core.script_generations.real_skyvern_page_ai import RealSkyvernPageAi+from skyvern.exceptions import SkyvernActionFailed@pytest.fixture@@ -84,26 +85,26 @@ class TestAiClickEmptyActions:return_value="mock_prompt",),):- with pytest.raises(Exception) as exc_info:+ with pytest.raises(SkyvernActionFailed) as exc_info:await real_skyvern_page_ai.ai_click(- selector=None, # No fallback selector+ selector=None,intention="Click the download button",)- # Should raise because no actions and no fallbackassert "AI click failed" in str(exc_info.value) or "AI could not find" in str(exc_info.value)@pytest.mark.asyncio- async def test_ai_click_raises_when_llm_call_fails_no_selector(+ async def test_ai_click_propagates_unknown_exception_when_no_selector(self, mock_page, mock_scraped_page, mock_context, mock_app):"""- When AI fails (exception) and there's no selector to fall back to,- ai_click should raise an exception.+ When the AI path raises a non-operational exception (LLM down, etc.) and+ there's no selector to fall back to, the original exception should+ propagate so the caller surfaces it as a 500, not a 422."""real_skyvern_page_ai = RealSkyvernPageAi(mock_scraped_page, mock_page)- mock_app.SINGLE_CLICK_AGENT_LLM_API_HANDLER = AsyncMock(side_effect=Exception("LLM error"))+ mock_app.SINGLE_CLICK_AGENT_LLM_API_HANDLER = AsyncMock(side_effect=RuntimeError("LLM provider down"))with (patch.object(real_skyvern_page_ai, "_refresh_scraped_page", new_callable=AsyncMock),@@ -117,13 +118,14 @@ class TestAiClickEmptyActions:return_value="mock_prompt",),):- with pytest.raises(Exception) as exc_info:+ with pytest.raises(RuntimeError) as exc_info:await real_skyvern_page_ai.ai_click(- selector=None, # No fallback selector+ selector=None,intention="Click the download button",)- assert "AI click failed" in str(exc_info.value)+ assert "LLM provider down" in str(exc_info.value)+ assert not isinstance(exc_info.value, SkyvernActionFailed)@pytest.mark.asyncioasync def test_ai_click_falls_back_to_selector_when_llm_returns_empty(diff --git a/tests/unit/test_run_sdk_action_failures.py b/tests/unit/test_run_sdk_action_failures.pynew file mode 100644index 000000000..1d87f2338--- /dev/null+++ b/tests/unit/test_run_sdk_action_failures.py@@ -0,0 +1,112 @@+from __future__ import annotations++from typing import Any+from unittest.mock import AsyncMock, MagicMock, patch++import pytest+from fastapi import HTTPException++from skyvern.exceptions import ScrapingFailed, SkyvernActionFailed+from skyvern.forge.sdk.routes.sdk import run_sdk_action+++def mock_request() -> Any:+ request = MagicMock()+ request.workflow_run_id = "wr_test"+ request.browser_session_id = None+ request.browser_address = None+ request.url = "https://example.com"+ request.action = MagicMock()+ request.action.type = "ai_click"+ request.action.selector = None+ request.action.intention = "Click the button"+ request.action.data = None+ request.action.timeout = 30000+ request.action.get_navigation_goal = MagicMock(return_value="Click the button")+ request.action.get_navigation_payload = MagicMock(return_value=None)+ return request+++def mock_organization() -> Any:+ org = MagicMock()+ org.organization_id = "o_test"+ return org+++def mock_app() -> Any:+ app = MagicMock()+ workflow_run = MagicMock(workflow_run_id="wr_test", workflow_id="w_test")+ workflow = MagicMock(workflow_id="w_test", workflow_permanent_id="wpid_test", title="t")+ task = MagicMock(task_id="tsk_test", organization_id="o_test", max_screenshot_scrolls=None)+ step = MagicMock(step_id="stp_test")+ app.DATABASE.workflow_runs.get_workflow_run = AsyncMock(return_value=workflow_run)+ app.DATABASE.workflows.get_workflow = AsyncMock(return_value=workflow)+ app.DATABASE.tasks.create_task = AsyncMock(return_value=task)+ app.DATABASE.tasks.create_step = AsyncMock(return_value=step)+ app.DATABASE.tasks.update_task = AsyncMock()+ app.DATABASE.observer.create_workflow_run_block = AsyncMock()+ app.WORKFLOW_CONTEXT_MANAGER.initialize_workflow_run_context = AsyncMock()+ return app+++async def test_handler_returns_422_when_action_raises_skyvern_action_failed(+ mock_request: Any, mock_organization: Any, mock_app: Any+) -> None:+ with (+ patch("skyvern.forge.sdk.routes.sdk.app", mock_app),+ patch("skyvern.forge.sdk.routes.sdk.skyvern_context") as mock_ctx,+ patch(+ "skyvern.core.script_generations.script_skyvern_page.ScriptSkyvernPage.create_scraped_page",+ new_callable=AsyncMock,+ side_effect=SkyvernActionFailed("AI click failed and no fallback selector available"),+ ),+ ):+ mock_ctx.ensure_context.return_value = MagicMock(request_id="req_test", tz_info=None, prompt=None)+ with pytest.raises(HTTPException) as exc_info:+ await run_sdk_action(mock_request, organization=mock_organization)++ assert exc_info.value.status_code == 422+ assert "AI click failed" in str(exc_info.value.detail)+ mock_app.DATABASE.tasks.update_task.assert_awaited()+++async def test_handler_returns_400_when_action_raises_scraping_failed(+ mock_request: Any, mock_organization: Any, mock_app: Any+) -> None:+ with (+ patch("skyvern.forge.sdk.routes.sdk.app", mock_app),+ patch("skyvern.forge.sdk.routes.sdk.skyvern_context") as mock_ctx,+ patch(+ "skyvern.core.script_generations.script_skyvern_page.ScriptSkyvernPage.create_scraped_page",+ new_callable=AsyncMock,+ side_effect=ScrapingFailed(reason="page is blank"),+ ),+ ):+ mock_ctx.ensure_context.return_value = MagicMock(request_id="req_test", tz_info=None, prompt=None)+ with pytest.raises(HTTPException) as exc_info:+ await run_sdk_action(mock_request, organization=mock_organization)++ assert exc_info.value.status_code == 400+++async def test_handler_propagates_unknown_exception(mock_request: Any, mock_organization: Any, mock_app: Any) -> None:+ with (+ patch("skyvern.forge.sdk.routes.sdk.app", mock_app),+ patch("skyvern.forge.sdk.routes.sdk.skyvern_context") as mock_ctx,+ patch(+ "skyvern.core.script_generations.script_skyvern_page.ScriptSkyvernPage.create_scraped_page",+ new_callable=AsyncMock,+ side_effect=RuntimeError("db connection pool exhausted"),+ ),+ ):+ mock_ctx.ensure_context.return_value = MagicMock(request_id="req_test", tz_info=None, prompt=None)+ with pytest.raises(RuntimeError):+ await run_sdk_action(mock_request, organization=mock_organization)++ mock_app.DATABASE.tasks.update_task.assert_awaited()