Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
fix(mcp): return a structured error for out-of-schema workflow_run_list arguments (#8392)
Aaron Perezdccf52e46636
diff --git a/skyvern/cli/mcp_tools/argument_validation.py b/skyvern/cli/mcp_tools/argument_validation.pyindex 43f4c6f28..de906bfae 100644--- a/skyvern/cli/mcp_tools/argument_validation.py+++ b/skyvern/cli/mcp_tools/argument_validation.py@@ -14,10 +14,17 @@ This middleware checks arguments against the tool's published input schemashort-circuits with a structured error when argument keys don't match. Becauseinvalid keys never reach dispatch, no raw validation error is logged, and themodel receives a clear message naming the unsupported and expected arguments-(the same ``make_result``/``make_error`` envelope every tool uses). Other-type-level mismatches still fall through to pydantic. A call with no-``arguments`` at all also falls through: pydantic handles it, rejecting only-when the tool has required parameters.+(the same ``make_result``/``make_error`` envelope every tool uses). A call+with no ``arguments`` at all also falls through: pydantic handles it,+rejecting only when the tool has required parameters.++Type-level mismatches the repair step doesn't cover (an out-of-range int, a+still-malformed list) still reach FastMCP's own pydantic validation inside+``call_next``. Those are deliberately not silently coerced — an ambiguous+shape should fail rather than be guessed at — but the resulting+``fastmcp.exceptions.ValidationError`` is caught here too and wrapped in the+same structured envelope, so the model always gets an actionable message+instead of the raw pydantic text."""from __future__ import annotations@@ -26,6 +33,7 @@ import jsonfrom typing import Anyimport structlog+from fastmcp.exceptions import ValidationError as FastMCPValidationErrorfrom fastmcp.server.middleware import CallNext, Middleware, MiddlewareContextfrom fastmcp.tools.tool import ToolResultfrom mcp.types import TextContent@@ -124,7 +132,15 @@ class MCPArgumentValidationMiddleware(Middleware):rejection = await self._reject_bad_arguments(context)if rejection is not None:return rejection- return await call_next(context)+ try:+ return await call_next(context)+ except FastMCPValidationError as exc:+ # Argument shapes/ranges that _reject_bad_arguments and _repair_argument_types don't+ # cover (e.g. an out-of-range int, a still-malformed list) reach here from FastMCP's+ # own pydantic validation inside tool dispatch. Without this, the caller gets the raw+ # exception text instead of the same structured envelope every other tool error uses.+ tool_name = getattr(context.message, "name", None)+ return _validation_error_result(tool_name, exc)async def _reject_bad_arguments(self, context: MiddlewareContext[Any]) -> ToolResult | None:tool_name = getattr(context.message, "name", None)@@ -198,4 +214,19 @@ def _rejection_result(return ToolResult(content=[TextContent(type="text", text=text)], structured_content=payload)+def _validation_error_result(tool_name: str | None, exc: FastMCPValidationError) -> ToolResult:+ name = tool_name or "unknown_tool"+ payload = make_result(+ name,+ ok=False,+ error=make_error(+ ErrorCode.INVALID_INPUT,+ str(exc),+ f"Check the argument types and values against {name}'s documented parameters.",+ ),+ )+ text = json.dumps(payload, ensure_ascii=False, default=str)+ return ToolResult(content=[TextContent(type="text", text=text)], structured_content=payload)++__all__ = ["MCPArgumentValidationMiddleware"]diff --git a/skyvern/forge/agent.py b/skyvern/forge/agent.pyindex 36fdae094..fd3706a0c 100644--- a/skyvern/forge/agent.py+++ b/skyvern/forge/agent.py@@ -44,8 +44,11 @@ from skyvern.errors.errors import (filter_to_user_defined_codes,)from skyvern.exceptions import (+ BrowserSessionAlreadyOccupiedError,+ BrowserSessionClosed,BrowserSessionDegraded,BrowserSessionNotFound,+ BrowserSessionOwnershipConflict,DownloadFileMaxWaitingTime,DownloadSaveIncompleteError,EmptyScrapePage,@@ -69,6 +72,7 @@ from skyvern.exceptions import (TaskAlreadyCanceled,TaskAlreadyTimeout,TaskNotFound,+ UnknownErrorWhileCreatingBrowserContext,UnsupportedActionType,UnsupportedTaskType,get_user_facing_exception_message,@@ -227,6 +231,7 @@ from skyvern.webeye.actions.parse_actions import ()from skyvern.webeye.actions.responses import ActionResult, ActionSuccessfrom skyvern.webeye.browser_engine import BrowserEngineSelection+from skyvern.webeye.browser_errors import BrowserTargetClosedErrorfrom skyvern.webeye.browser_state import BrowserState, get_browser_state_diagnosticfrom skyvern.webeye.cdp_download_interceptor import (consume_unsolicited_download_error_for_context,@@ -2903,6 +2908,40 @@ class ForgeAgent:list_files_before=list_files_before,)return step, detailed_output, None+ except (+ UnknownErrorWhileCreatingBrowserContext,+ BrowserSessionAlreadyOccupiedError,+ BrowserSessionClosed,+ BrowserSessionOwnershipConflict,+ BrowserTargetClosedError,+ ) as e:+ # UnknownErrorWhileCreatingBrowserContext wraps *any* exception raised during+ # browser-context creation (BrowserFactory.create_browser_context's `except+ # BaseException`), so unlike its three siblings above it isn't inherently a known+ # condition. Only mute it when the wrapped cause is itself one of Skyvern's own+ # named exceptions (e.g. a proxy-capacity error) -- an unrecognized cause (a raw+ # AttributeError/TypeError from a real bug) stays on the paging path.+ if isinstance(e, UnknownErrorWhileCreatingBrowserContext) and not isinstance(e.__cause__, SkyvernException):+ LOG.exception("Got an unexpected exception in step, marking task as failed")+ else:+ LOG.warning("Recognized browser/session failure, marking the task as failed", exc_info=True)++ failure_reason = get_user_facing_exception_message(e)++ is_task_marked_as_failed = await self.fail_task(task, step, failure_reason, browser_state, exception=e)+ if is_task_marked_as_failed:+ await self.clean_up_task(+ task=task,+ last_step=step,+ api_key=api_key,+ close_browser_on_completion=close_browser_on_completion,+ browser_session_id=browser_session_id,+ download_suffix=task_block.download_suffix if task_block else None,+ list_files_before=list_files_before,+ )+ else:+ LOG.warning("Task isn't marked as failed, after browser/session failure. NOT clean up the task")+ return step, detailed_output, Noneexcept Exception as e:LOG.exception("Got an unexpected exception in step, marking task as failed")@@ -6395,9 +6434,11 @@ class ForgeAgent:)return None- async def get_failure_reason_for_task(self, task: Task) -> str | None:+ async def get_failure_reason_for_task(self, task: Task) -> str:"""Find the TerminateAction for the task and return the reasoning.+ TaskStatus.terminated requires a failure_reason (see validate_update), so this always+ returns a non-empty string even when no reasoning can be recovered.# TODO (kerem): Also return meaningful exceptions when we add them [WYV-311]"""steps = await app.DATABASE.tasks.get_task_steps(@@ -6413,13 +6454,13 @@ class ForgeAgent:if step.output.actions_and_results:for action, action_results in step.output.actions_and_results:if action.action_type == ActionType.TERMINATE:- return action.reasoning+ return action.reasoning or "Task was terminated without a stated reason."LOG.error("Failed to find failure reasoning for task",task_id=task.task_id,)- return None+ return "Task was terminated, but Skyvern could not determine why."@traced(name="skyvern.agent.cleanup")async def clean_up_task(diff --git a/skyvern/webeye/utils/dom.py b/skyvern/webeye/utils/dom.pyindex e535224e6..7587aedcb 100644--- a/skyvern/webeye/utils/dom.py+++ b/skyvern/webeye/utils/dom.py@@ -1171,6 +1171,8 @@ class SkyvernElement:try:await handler_utils.input_sequentially(self.get_locator(), text, timeout=default_timeout)except Exception as exc:+ if is_engine_error(exc, self._engine_selection) and is_incompatible_text_input_error(exc):+ raise InvalidElementForTextInput(element_id=self.get_id(), tag_name=self.get_tag_name())if not is_engine_timeout_error(exc, self._engine_selection):raiseawait self._classify_typing_timeout(exc)diff --git a/tests/unit/test_agent_step_characterization.py b/tests/unit/test_agent_step_characterization.pyindex d1599b549..f79d19542 100644--- a/tests/unit/test_agent_step_characterization.py+++ b/tests/unit/test_agent_step_characterization.py@@ -11,6 +11,7 @@ from zoneinfo import ZoneInfoimport pytestfrom skyvern.exceptions import NoTOTPVerificationCodeFound+from skyvern.forge import appfrom skyvern.forge.agent import ForgeAgent, StepPromptResultfrom skyvern.forge.sdk.core import skyvern_contextfrom skyvern.forge.sdk.core.skyvern_context import SkyvernContext@@ -18,6 +19,7 @@ from skyvern.forge.sdk.models import Step, StepStatusfrom skyvern.forge.sdk.schemas.organizations import Organizationfrom skyvern.forge.sdk.schemas.tasks import Taskfrom skyvern.forge.sdk.workflow.models.block import BaseTaskBlock, FileDownloadBlock+from skyvern.schemas.steps import AgentStepOutputfrom skyvern.webeye.actions.action_types import ActionTypefrom skyvern.webeye.actions.actions import (Action,@@ -25,6 +27,7 @@ from skyvern.webeye.actions.actions import (CompleteAction,DownloadFileAction,ExtractAction,+ TerminateAction,WaitAction,)from skyvern.webeye.actions.models import DetailedAgentStepOutput@@ -447,3 +450,39 @@ async def test_parallel_verification_marks_speculative_original_status(monkeypatassert step.status == StepStatus.completedassert step.speculative_original_status == StepStatus.completed+++# SKY-15xxx: TaskStatus.terminated requires a non-empty failure_reason (Task.validate_update).+# get_failure_reason_for_task is the only source of that reason for a terminated task; either+# empty path below used to return None, which handle_completed_step then handed straight to+# update_task, tripping the invariant and crashing the step as an "unexpected exception".+async def test_get_failure_reason_for_task_falls_back_when_terminate_reasoning_is_empty(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ now = datetime.now(UTC)+ organization = make_organization(now)+ task = make_task(now, organization)+ terminate = TerminateAction(reasoning=None, organization_id=organization.organization_id, task_id=task.task_id)+ output = AgentStepOutput(actions_and_results=[(terminate, [])])+ step = make_step(now, task, step_id="step-0", status=StepStatus.completed, order=0, output=output)+ monkeypatch.setattr(app.DATABASE.tasks, "get_task_steps", AsyncMock(return_value=[step]))++ reason = await ForgeAgent().get_failure_reason_for_task(task)++ assert reason+++async def test_get_failure_reason_for_task_falls_back_when_no_terminate_action_found(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ now = datetime.now(UTC)+ organization = make_organization(now)+ task = make_task(now, organization)+ step = make_step(now, task, step_id="step-0", status=StepStatus.completed, order=0, output=None)+ monkeypatch.setattr(app.DATABASE.tasks, "get_task_steps", AsyncMock(return_value=[step]))++ reason = await ForgeAgent().get_failure_reason_for_task(task)++ assert reasondiff --git a/tests/unit/test_execute_step_exception_handling.py b/tests/unit/test_execute_step_exception_handling.pyindex fe8c50db7..b726a0424 100644--- a/tests/unit/test_execute_step_exception_handling.py+++ b/tests/unit/test_execute_step_exception_handling.py@@ -35,6 +35,9 @@ import pytestfrom skyvern.config import settings as agent_settingsfrom skyvern.exceptions import (+ BrowserSessionAlreadyOccupiedError,+ BrowserSessionClosed,+ BrowserSessionOwnershipConflict,FailedToNavigateToUrl,FailedToParseActionInstruction,FailedToSendWebhook,@@ -45,6 +48,7 @@ from skyvern.exceptions import (StepUnableToExecuteError,TaskAlreadyCanceled,TaskAlreadyTimeout,+ UnknownErrorWhileCreatingBrowserContext,UnsupportedActionType,UnsupportedTaskType,)@@ -52,6 +56,7 @@ from skyvern.forge.agent import ForgeAgentfrom skyvern.forge.sdk.core import skyvern_contextfrom skyvern.forge.sdk.core.skyvern_context import SkyvernContextfrom skyvern.forge.sdk.models import StepStatus+from skyvern.webeye.browser_errors import BrowserTargetClosedErrorfrom tests.unit.helpers import make_organization, make_step, make_task# ``clean_up_task``'s real signature defaults; handlers that omit these kwargs get them.@@ -248,6 +253,53 @@ CLEANUP_CASES: list[CleanupHandlerCase] = [effective_final_screenshot=True,cleanup_gated_on_fail_task=True,),+ # SKY-15xxx: these four are Skyvern's own named browser/session exceptions (already+ # anticipated, unlike a true "unexpected" error) that used to fall through to the+ # generic_exception handler above and page P1 on every occurrence. They now share a+ # dedicated LOG.warning handler with the same fail_task/clean_up_task shape as+ # generic_exception, just without the error-level page. fail_task/clean_up_task shape+ # is identical here regardless of whether the wrapped cause is recognized (see the+ # dedicated log-level tests below for that split).+ CleanupHandlerCase(+ id="unknown_error_creating_browser_context",+ exc_factory=lambda: UnknownErrorWhileCreatingBrowserContext("chromium", RuntimeError("boom")),+ fail_task_called=True,+ effective_webhook=True,+ effective_final_screenshot=True,+ cleanup_gated_on_fail_task=True,+ ),+ CleanupHandlerCase(+ id="browser_session_already_occupied",+ exc_factory=lambda: BrowserSessionAlreadyOccupiedError("pbs-1", "tsk-2"),+ fail_task_called=True,+ effective_webhook=True,+ effective_final_screenshot=True,+ cleanup_gated_on_fail_task=True,+ ),+ CleanupHandlerCase(+ id="browser_session_closed",+ exc_factory=lambda: BrowserSessionClosed("pbs-1"),+ fail_task_called=True,+ effective_webhook=True,+ effective_final_screenshot=True,+ cleanup_gated_on_fail_task=True,+ ),+ CleanupHandlerCase(+ id="browser_session_ownership_conflict",+ exc_factory=lambda: BrowserSessionOwnershipConflict("pbs-1"),+ fail_task_called=True,+ effective_webhook=True,+ effective_final_screenshot=True,+ cleanup_gated_on_fail_task=True,+ ),+ CleanupHandlerCase(+ id="browser_target_closed",+ exc_factory=lambda: BrowserTargetClosedError("session closed underneath the run"),+ fail_task_called=True,+ effective_webhook=True,+ effective_final_screenshot=True,+ cleanup_gated_on_fail_task=True,+ ),]@@ -289,6 +341,89 @@ async def test_cleanup_gating_when_fail_task_reports_not_failed(assert outcome.cleanup_called is not case.cleanup_gated_on_fail_task+async def test_unknown_browser_context_error_pages_p1_for_unrecognized_cause(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ """BrowserFactory.create_browser_context wraps *any* exception, so a cause that isn't+ one of Skyvern's own named exceptions (e.g. a real bug's AttributeError) is not an+ anticipated browser/session failure -- it must keep the error-level page, not go quiet.+ """+ log_mock = MagicMock()+ monkeypatch.setattr("skyvern.forge.agent.LOG", log_mock)++ cause = AttributeError("'NoneType' object has no attribute 'frame'")+ exc = UnknownErrorWhileCreatingBrowserContext("chromium", cause)+ exc.__cause__ = cause # mirrors `raise ... from e` at the real call site++ outcome = await _drive_execute_step(monkeypatch, exc, fail_task_result=True)++ assert outcome.raised is None+ assert outcome.fail_task_called is True+ assert outcome.cleanup_called is True+ log_mock.exception.assert_called_once_with("Got an unexpected exception in step, marking task as failed")+ log_mock.warning.assert_not_called()+++async def test_unknown_browser_context_error_stays_quiet_for_recognized_cause(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ """A cause that IS one of Skyvern's own named exceptions (e.g. a proxy-capacity error+ surfacing as MissingBrowserStatePage) is the anticipated case this handler exists for --+ it must stay on the quiet, non-paging path.+ """+ log_mock = MagicMock()+ monkeypatch.setattr("skyvern.forge.agent.LOG", log_mock)++ cause = MissingBrowserStatePage(task_id="task-123")+ exc = UnknownErrorWhileCreatingBrowserContext("chromium", cause)+ exc.__cause__ = cause++ outcome = await _drive_execute_step(monkeypatch, exc, fail_task_result=True)++ assert outcome.raised is None+ assert outcome.fail_task_called is True+ assert outcome.cleanup_called is True+ log_mock.warning.assert_called_once()+ log_mock.exception.assert_not_called()+++ "exc_factory",+ [+ lambda: BrowserSessionAlreadyOccupiedError("pbs-1", "tsk-2"),+ lambda: BrowserSessionClosed("pbs-1"),+ lambda: BrowserSessionOwnershipConflict("pbs-1"),+ lambda: BrowserTargetClosedError("session closed underneath the run"),+ ],+ ids=[+ "browser_session_already_occupied",+ "browser_session_closed",+ "browser_session_ownership_conflict",+ "browser_target_closed",+ ],+)+async def test_direct_session_exceptions_stay_on_the_quiet_path(+ monkeypatch: pytest.MonkeyPatch, exc_factory: Callable[[], BaseException]+) -> None:+ """These four are Skyvern's own named browser/session exceptions -- unlike+ UnknownErrorWhileCreatingBrowserContext's conditional handling, they're unconditionally+ quiet. CLEANUP_CASES pins fail_task/clean_up_task/webhook/screenshot, which stay+ identical to generic_exception's -- only the log level tells these apart, so pin that+ directly: dropping any one of the four from the tuple must fail this test.+ """+ log_mock = MagicMock()+ monkeypatch.setattr("skyvern.forge.agent.LOG", log_mock)++ outcome = await _drive_execute_step(monkeypatch, exc_factory(), fail_task_result=True)++ assert outcome.raised is None+ log_mock.warning.assert_called_once()+ log_mock.exception.assert_not_called()++@pytest.mark.asyncioasync def test_step_unable_to_execute_reraises_without_cleanup(monkeypatch: pytest.MonkeyPatch) -> None:"""StepUnableToExecuteError propagates out of execute_step; no fail_task, no cleanup."""diff --git a/tests/unit/test_incompatible_element_guards.py b/tests/unit/test_incompatible_element_guards.pyindex cefef1f41..9fa18dd16 100644--- a/tests/unit/test_incompatible_element_guards.py+++ b/tests/unit/test_incompatible_element_guards.py@@ -161,6 +161,39 @@ async def test_input_clear_classifies_incompatible_element(monkeypatch: pytest.Mawait element.input_clear()+def _make_non_stale_locator() -> MagicMock:+ locator = MagicMock()+ locator.count = AsyncMock(return_value=1) # refresh_locator_if_stale sees a unique match+ return locator+++async def test_input_sequentially_classifies_incompatible_element(monkeypatch: pytest.MonkeyPatch) -> None:+ # input_sequentially's underlying strategy_aware_input can call locator.fill() directly (the+ # long-text "fast prefix" path); until this fix that raw Playwright error skipped the same+ # classification input_fill/input_clear already apply and escaped unclassified (SKY-15219).+ element = _make_element("huk-input-field", _make_non_stale_locator())++ async def _raise(*args: object, **kwargs: object) -> None:+ raise PlaywrightError(_NOT_INPUT_ERROR)++ monkeypatch.setattr("skyvern.webeye.utils.dom.handler_utils.input_sequentially", _raise)+ with pytest.raises(InvalidElementForTextInput):+ await element.input_sequentially("hello")+++async def test_input_sequentially_reraises_unrelated_errors(monkeypatch: pytest.MonkeyPatch) -> None:+ element = _make_element("input", _make_non_stale_locator())++ async def _raise(*args: object, **kwargs: object) -> None:+ raise PlaywrightError(_TIMEOUT_ERROR)++ monkeypatch.setattr("skyvern.webeye.utils.dom.handler_utils.input_sequentially", _raise)+ with pytest.raises(PlaywrightError):+ await element.input_sequentially("hello")++# --------------------------------------------------------------------------- ## _strict_date_mask_order / _canonical_iso_date — deterministic date canonicalization.# A live <input type=date> takes only YYYY-MM-DD; a locale value is canonicalized only when thediff --git a/tests/unit/test_mcp_argument_validation.py b/tests/unit/test_mcp_argument_validation.pyindex ca1b81e43..8b44ee2f6 100644--- a/tests/unit/test_mcp_argument_validation.py+++ b/tests/unit/test_mcp_argument_validation.py@@ -212,3 +212,34 @@ async def test_type_repair_preserves_structured_missing_key_rejection() -> None:assert error["code"] == "INVALID_INPUT"assert error["details"]["unsupported_arguments"] == []assert error["details"]["missing_required_arguments"] == ["workflow_id"]+++ ("arguments", "expected_field"),+ [+ # SKY-15319: page below the tool's documented 1-based minimum.+ ({"workflow_id": "wpid_test", "page": 0}, "page"),+ # SKY-14997: a single status value sent as a bare string instead of a list. Not+ # comma-separated, so the narrow CSV repair correctly leaves it for validation.+ ({"workflow_id": "wpid_test", "status": "completed"}, "status"),+ ],+)+async def test_out_of_schema_value_gets_structured_error_instead_of_raising(+ monkeypatch: pytest.MonkeyPatch,+ arguments: dict,+ expected_field: str,+) -> None:+ list_runs = AsyncMock(return_value=[])+ monkeypatch.setattr(mcp_workflow, "list_workflow_runs_raw", list_runs)++ # Regression proof: before the fix, FastMCP's own pydantic validation raised+ # fastmcp.exceptions.ValidationError out of call_next, uncaught by this middleware.+ payload = await _call("skyvern_workflow_run_list", arguments)++ assert payload["ok"] is False+ error = payload["error"]+ assert error["code"] == "INVALID_INPUT"+ assert expected_field in error["message"]+ assert "skyvern_workflow_run_list" in error["hint"]+ list_runs.assert_not_awaited()