Skip to content

Automate browser based workflows with AI

fix(mcp): return a structured error for out-of-schema workflow_run_list arguments (#8392)

Aaron Perezdccf52e46636
diff --git a/skyvern/cli/mcp_tools/argument_validation.py b/skyvern/cli/mcp_tools/argument_validation.py
index 43f4c6f28..de906bfae 100644
--- a/skyvern/cli/mcp_tools/argument_validation.py
+++ b/skyvern/cli/mcp_tools/argument_validation.py
@@ -14,10 +14,17 @@ This middleware checks arguments against the tool's published input schema
short-circuits with a structured error when argument keys don't match. Because
invalid keys never reach dispatch, no raw validation error is logged, and the
model receives a clear message naming the unsupported and expected arguments
-(the same ``make_result``/``make_error`` envelope every tool uses). Other
-type-level mismatches still fall through to pydantic. A call with no
-``arguments`` at all also falls through: pydantic handles it, rejecting only
-when the tool has required parameters.
+(the same ``make_result``/``make_error`` envelope every tool uses). A call
+with no ``arguments`` at all also falls through: pydantic handles it,
+rejecting only when the tool has required parameters.
+
+Type-level mismatches the repair step doesn't cover (an out-of-range int, a
+still-malformed list) still reach FastMCP's own pydantic validation inside
+``call_next``. Those are deliberately not silently coerced — an ambiguous
+shape should fail rather than be guessed at — but the resulting
+``fastmcp.exceptions.ValidationError`` is caught here too and wrapped in the
+same structured envelope, so the model always gets an actionable message
+instead of the raw pydantic text.
"""
from __future__ import annotations
@@ -26,6 +33,7 @@ import json
from typing import Any
import structlog
+from fastmcp.exceptions import ValidationError as FastMCPValidationError
from fastmcp.server.middleware import CallNext, Middleware, MiddlewareContext
from fastmcp.tools.tool import ToolResult
from mcp.types import TextContent
@@ -124,7 +132,15 @@ class MCPArgumentValidationMiddleware(Middleware):
rejection = await self._reject_bad_arguments(context)
if rejection is not None:
return rejection
- return await call_next(context)
+ try:
+ return await call_next(context)
+ except FastMCPValidationError as exc:
+ # Argument shapes/ranges that _reject_bad_arguments and _repair_argument_types don't
+ # cover (e.g. an out-of-range int, a still-malformed list) reach here from FastMCP's
+ # own pydantic validation inside tool dispatch. Without this, the caller gets the raw
+ # exception text instead of the same structured envelope every other tool error uses.
+ tool_name = getattr(context.message, "name", None)
+ return _validation_error_result(tool_name, exc)
async def _reject_bad_arguments(self, context: MiddlewareContext[Any]) -> ToolResult | None:
tool_name = getattr(context.message, "name", None)
@@ -198,4 +214,19 @@ def _rejection_result(
return ToolResult(content=[TextContent(type="text", text=text)], structured_content=payload)
+def _validation_error_result(tool_name: str | None, exc: FastMCPValidationError) -> ToolResult:
+ name = tool_name or "unknown_tool"
+ payload = make_result(
+ name,
+ ok=False,
+ error=make_error(
+ ErrorCode.INVALID_INPUT,
+ str(exc),
+ f"Check the argument types and values against {name}'s documented parameters.",
+ ),
+ )
+ text = json.dumps(payload, ensure_ascii=False, default=str)
+ return ToolResult(content=[TextContent(type="text", text=text)], structured_content=payload)
+
+
__all__ = ["MCPArgumentValidationMiddleware"]
diff --git a/skyvern/forge/agent.py b/skyvern/forge/agent.py
index 36fdae094..fd3706a0c 100644
--- a/skyvern/forge/agent.py
+++ b/skyvern/forge/agent.py
@@ -44,8 +44,11 @@ from skyvern.errors.errors import (
filter_to_user_defined_codes,
)
from skyvern.exceptions import (
+ BrowserSessionAlreadyOccupiedError,
+ BrowserSessionClosed,
BrowserSessionDegraded,
BrowserSessionNotFound,
+ BrowserSessionOwnershipConflict,
DownloadFileMaxWaitingTime,
DownloadSaveIncompleteError,
EmptyScrapePage,
@@ -69,6 +72,7 @@ from skyvern.exceptions import (
TaskAlreadyCanceled,
TaskAlreadyTimeout,
TaskNotFound,
+ UnknownErrorWhileCreatingBrowserContext,
UnsupportedActionType,
UnsupportedTaskType,
get_user_facing_exception_message,
@@ -227,6 +231,7 @@ from skyvern.webeye.actions.parse_actions import (
)
from skyvern.webeye.actions.responses import ActionResult, ActionSuccess
from skyvern.webeye.browser_engine import BrowserEngineSelection
+from skyvern.webeye.browser_errors import BrowserTargetClosedError
from skyvern.webeye.browser_state import BrowserState, get_browser_state_diagnostic
from skyvern.webeye.cdp_download_interceptor import (
consume_unsolicited_download_error_for_context,
@@ -2903,6 +2908,40 @@ class ForgeAgent:
list_files_before=list_files_before,
)
return step, detailed_output, None
+ except (
+ UnknownErrorWhileCreatingBrowserContext,
+ BrowserSessionAlreadyOccupiedError,
+ BrowserSessionClosed,
+ BrowserSessionOwnershipConflict,
+ BrowserTargetClosedError,
+ ) as e:
+ # UnknownErrorWhileCreatingBrowserContext wraps *any* exception raised during
+ # browser-context creation (BrowserFactory.create_browser_context's `except
+ # BaseException`), so unlike its three siblings above it isn't inherently a known
+ # condition. Only mute it when the wrapped cause is itself one of Skyvern's own
+ # named exceptions (e.g. a proxy-capacity error) -- an unrecognized cause (a raw
+ # AttributeError/TypeError from a real bug) stays on the paging path.
+ if isinstance(e, UnknownErrorWhileCreatingBrowserContext) and not isinstance(e.__cause__, SkyvernException):
+ LOG.exception("Got an unexpected exception in step, marking task as failed")
+ else:
+ LOG.warning("Recognized browser/session failure, marking the task as failed", exc_info=True)
+
+ failure_reason = get_user_facing_exception_message(e)
+
+ is_task_marked_as_failed = await self.fail_task(task, step, failure_reason, browser_state, exception=e)
+ if is_task_marked_as_failed:
+ await self.clean_up_task(
+ task=task,
+ last_step=step,
+ api_key=api_key,
+ close_browser_on_completion=close_browser_on_completion,
+ browser_session_id=browser_session_id,
+ download_suffix=task_block.download_suffix if task_block else None,
+ list_files_before=list_files_before,
+ )
+ else:
+ LOG.warning("Task isn't marked as failed, after browser/session failure. NOT clean up the task")
+ return step, detailed_output, None
except Exception as e:
LOG.exception("Got an unexpected exception in step, marking task as failed")
@@ -6395,9 +6434,11 @@ class ForgeAgent:
)
return None
- async def get_failure_reason_for_task(self, task: Task) -> str | None:
+ async def get_failure_reason_for_task(self, task: Task) -> str:
"""
Find the TerminateAction for the task and return the reasoning.
+ TaskStatus.terminated requires a failure_reason (see validate_update), so this always
+ returns a non-empty string even when no reasoning can be recovered.
# TODO (kerem): Also return meaningful exceptions when we add them [WYV-311]
"""
steps = await app.DATABASE.tasks.get_task_steps(
@@ -6413,13 +6454,13 @@ class ForgeAgent:
if step.output.actions_and_results:
for action, action_results in step.output.actions_and_results:
if action.action_type == ActionType.TERMINATE:
- return action.reasoning
+ return action.reasoning or "Task was terminated without a stated reason."
LOG.error(
"Failed to find failure reasoning for task",
task_id=task.task_id,
)
- return None
+ return "Task was terminated, but Skyvern could not determine why."
@traced(name="skyvern.agent.cleanup")
async def clean_up_task(
diff --git a/skyvern/webeye/utils/dom.py b/skyvern/webeye/utils/dom.py
index e535224e6..7587aedcb 100644
--- a/skyvern/webeye/utils/dom.py
+++ b/skyvern/webeye/utils/dom.py
@@ -1171,6 +1171,8 @@ class SkyvernElement:
try:
await handler_utils.input_sequentially(self.get_locator(), text, timeout=default_timeout)
except Exception as exc:
+ if is_engine_error(exc, self._engine_selection) and is_incompatible_text_input_error(exc):
+ raise InvalidElementForTextInput(element_id=self.get_id(), tag_name=self.get_tag_name())
if not is_engine_timeout_error(exc, self._engine_selection):
raise
await self._classify_typing_timeout(exc)
diff --git a/tests/unit/test_agent_step_characterization.py b/tests/unit/test_agent_step_characterization.py
index d1599b549..f79d19542 100644
--- a/tests/unit/test_agent_step_characterization.py
+++ b/tests/unit/test_agent_step_characterization.py
@@ -11,6 +11,7 @@ from zoneinfo import ZoneInfo
import pytest
from skyvern.exceptions import NoTOTPVerificationCodeFound
+from skyvern.forge import app
from skyvern.forge.agent import ForgeAgent, StepPromptResult
from skyvern.forge.sdk.core import skyvern_context
from skyvern.forge.sdk.core.skyvern_context import SkyvernContext
@@ -18,6 +19,7 @@ from skyvern.forge.sdk.models import Step, StepStatus
from skyvern.forge.sdk.schemas.organizations import Organization
from skyvern.forge.sdk.schemas.tasks import Task
from skyvern.forge.sdk.workflow.models.block import BaseTaskBlock, FileDownloadBlock
+from skyvern.schemas.steps import AgentStepOutput
from skyvern.webeye.actions.action_types import ActionType
from skyvern.webeye.actions.actions import (
Action,
@@ -25,6 +27,7 @@ from skyvern.webeye.actions.actions import (
CompleteAction,
DownloadFileAction,
ExtractAction,
+ TerminateAction,
WaitAction,
)
from skyvern.webeye.actions.models import DetailedAgentStepOutput
@@ -447,3 +450,39 @@ async def test_parallel_verification_marks_speculative_original_status(monkeypat
assert step.status == StepStatus.completed
assert step.speculative_original_status == StepStatus.completed
+
+
+# SKY-15xxx: TaskStatus.terminated requires a non-empty failure_reason (Task.validate_update).
+# get_failure_reason_for_task is the only source of that reason for a terminated task; either
+# empty path below used to return None, which handle_completed_step then handed straight to
+# update_task, tripping the invariant and crashing the step as an "unexpected exception".
+async def test_get_failure_reason_for_task_falls_back_when_terminate_reasoning_is_empty(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ now = datetime.now(UTC)
+ organization = make_organization(now)
+ task = make_task(now, organization)
+ terminate = TerminateAction(reasoning=None, organization_id=organization.organization_id, task_id=task.task_id)
+ output = AgentStepOutput(actions_and_results=[(terminate, [])])
+ step = make_step(now, task, step_id="step-0", status=StepStatus.completed, order=0, output=output)
+ monkeypatch.setattr(app.DATABASE.tasks, "get_task_steps", AsyncMock(return_value=[step]))
+
+ reason = await ForgeAgent().get_failure_reason_for_task(task)
+
+ assert reason
+
+
+async def test_get_failure_reason_for_task_falls_back_when_no_terminate_action_found(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ now = datetime.now(UTC)
+ organization = make_organization(now)
+ task = make_task(now, organization)
+ step = make_step(now, task, step_id="step-0", status=StepStatus.completed, order=0, output=None)
+ monkeypatch.setattr(app.DATABASE.tasks, "get_task_steps", AsyncMock(return_value=[step]))
+
+ reason = await ForgeAgent().get_failure_reason_for_task(task)
+
+ assert reason
diff --git a/tests/unit/test_execute_step_exception_handling.py b/tests/unit/test_execute_step_exception_handling.py
index fe8c50db7..b726a0424 100644
--- a/tests/unit/test_execute_step_exception_handling.py
+++ b/tests/unit/test_execute_step_exception_handling.py
@@ -35,6 +35,9 @@ import pytest
from skyvern.config import settings as agent_settings
from skyvern.exceptions import (
+ BrowserSessionAlreadyOccupiedError,
+ BrowserSessionClosed,
+ BrowserSessionOwnershipConflict,
FailedToNavigateToUrl,
FailedToParseActionInstruction,
FailedToSendWebhook,
@@ -45,6 +48,7 @@ from skyvern.exceptions import (
StepUnableToExecuteError,
TaskAlreadyCanceled,
TaskAlreadyTimeout,
+ UnknownErrorWhileCreatingBrowserContext,
UnsupportedActionType,
UnsupportedTaskType,
)
@@ -52,6 +56,7 @@ from skyvern.forge.agent import ForgeAgent
from skyvern.forge.sdk.core import skyvern_context
from skyvern.forge.sdk.core.skyvern_context import SkyvernContext
from skyvern.forge.sdk.models import StepStatus
+from skyvern.webeye.browser_errors import BrowserTargetClosedError
from tests.unit.helpers import make_organization, make_step, make_task
# ``clean_up_task``'s real signature defaults; handlers that omit these kwargs get them.
@@ -248,6 +253,53 @@ CLEANUP_CASES: list[CleanupHandlerCase] = [
effective_final_screenshot=True,
cleanup_gated_on_fail_task=True,
),
+ # SKY-15xxx: these four are Skyvern's own named browser/session exceptions (already
+ # anticipated, unlike a true "unexpected" error) that used to fall through to the
+ # generic_exception handler above and page P1 on every occurrence. They now share a
+ # dedicated LOG.warning handler with the same fail_task/clean_up_task shape as
+ # generic_exception, just without the error-level page. fail_task/clean_up_task shape
+ # is identical here regardless of whether the wrapped cause is recognized (see the
+ # dedicated log-level tests below for that split).
+ CleanupHandlerCase(
+ id="unknown_error_creating_browser_context",
+ exc_factory=lambda: UnknownErrorWhileCreatingBrowserContext("chromium", RuntimeError("boom")),
+ fail_task_called=True,
+ effective_webhook=True,
+ effective_final_screenshot=True,
+ cleanup_gated_on_fail_task=True,
+ ),
+ CleanupHandlerCase(
+ id="browser_session_already_occupied",
+ exc_factory=lambda: BrowserSessionAlreadyOccupiedError("pbs-1", "tsk-2"),
+ fail_task_called=True,
+ effective_webhook=True,
+ effective_final_screenshot=True,
+ cleanup_gated_on_fail_task=True,
+ ),
+ CleanupHandlerCase(
+ id="browser_session_closed",
+ exc_factory=lambda: BrowserSessionClosed("pbs-1"),
+ fail_task_called=True,
+ effective_webhook=True,
+ effective_final_screenshot=True,
+ cleanup_gated_on_fail_task=True,
+ ),
+ CleanupHandlerCase(
+ id="browser_session_ownership_conflict",
+ exc_factory=lambda: BrowserSessionOwnershipConflict("pbs-1"),
+ fail_task_called=True,
+ effective_webhook=True,
+ effective_final_screenshot=True,
+ cleanup_gated_on_fail_task=True,
+ ),
+ CleanupHandlerCase(
+ id="browser_target_closed",
+ exc_factory=lambda: BrowserTargetClosedError("session closed underneath the run"),
+ fail_task_called=True,
+ effective_webhook=True,
+ effective_final_screenshot=True,
+ cleanup_gated_on_fail_task=True,
+ ),
]
@@ -289,6 +341,89 @@ async def test_cleanup_gating_when_fail_task_reports_not_failed(
assert outcome.cleanup_called is not case.cleanup_gated_on_fail_task
+async def test_unknown_browser_context_error_pages_p1_for_unrecognized_cause(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ """BrowserFactory.create_browser_context wraps *any* exception, so a cause that isn't
+ one of Skyvern's own named exceptions (e.g. a real bug's AttributeError) is not an
+ anticipated browser/session failure -- it must keep the error-level page, not go quiet.
+ """
+ log_mock = MagicMock()
+ monkeypatch.setattr("skyvern.forge.agent.LOG", log_mock)
+
+ cause = AttributeError("'NoneType' object has no attribute 'frame'")
+ exc = UnknownErrorWhileCreatingBrowserContext("chromium", cause)
+ exc.__cause__ = cause # mirrors `raise ... from e` at the real call site
+
+ outcome = await _drive_execute_step(monkeypatch, exc, fail_task_result=True)
+
+ assert outcome.raised is None
+ assert outcome.fail_task_called is True
+ assert outcome.cleanup_called is True
+ log_mock.exception.assert_called_once_with("Got an unexpected exception in step, marking task as failed")
+ log_mock.warning.assert_not_called()
+
+
+async def test_unknown_browser_context_error_stays_quiet_for_recognized_cause(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ """A cause that IS one of Skyvern's own named exceptions (e.g. a proxy-capacity error
+ surfacing as MissingBrowserStatePage) is the anticipated case this handler exists for --
+ it must stay on the quiet, non-paging path.
+ """
+ log_mock = MagicMock()
+ monkeypatch.setattr("skyvern.forge.agent.LOG", log_mock)
+
+ cause = MissingBrowserStatePage(task_id="task-123")
+ exc = UnknownErrorWhileCreatingBrowserContext("chromium", cause)
+ exc.__cause__ = cause
+
+ outcome = await _drive_execute_step(monkeypatch, exc, fail_task_result=True)
+
+ assert outcome.raised is None
+ assert outcome.fail_task_called is True
+ assert outcome.cleanup_called is True
+ log_mock.warning.assert_called_once()
+ log_mock.exception.assert_not_called()
+
+
+ "exc_factory",
+ [
+ lambda: BrowserSessionAlreadyOccupiedError("pbs-1", "tsk-2"),
+ lambda: BrowserSessionClosed("pbs-1"),
+ lambda: BrowserSessionOwnershipConflict("pbs-1"),
+ lambda: BrowserTargetClosedError("session closed underneath the run"),
+ ],
+ ids=[
+ "browser_session_already_occupied",
+ "browser_session_closed",
+ "browser_session_ownership_conflict",
+ "browser_target_closed",
+ ],
+)
+async def test_direct_session_exceptions_stay_on_the_quiet_path(
+ monkeypatch: pytest.MonkeyPatch, exc_factory: Callable[[], BaseException]
+) -> None:
+ """These four are Skyvern's own named browser/session exceptions -- unlike
+ UnknownErrorWhileCreatingBrowserContext's conditional handling, they're unconditionally
+ quiet. CLEANUP_CASES pins fail_task/clean_up_task/webhook/screenshot, which stay
+ identical to generic_exception's -- only the log level tells these apart, so pin that
+ directly: dropping any one of the four from the tuple must fail this test.
+ """
+ log_mock = MagicMock()
+ monkeypatch.setattr("skyvern.forge.agent.LOG", log_mock)
+
+ outcome = await _drive_execute_step(monkeypatch, exc_factory(), fail_task_result=True)
+
+ assert outcome.raised is None
+ log_mock.warning.assert_called_once()
+ log_mock.exception.assert_not_called()
+
+
@pytest.mark.asyncio
async def test_step_unable_to_execute_reraises_without_cleanup(monkeypatch: pytest.MonkeyPatch) -> None:
"""StepUnableToExecuteError propagates out of execute_step; no fail_task, no cleanup."""
diff --git a/tests/unit/test_incompatible_element_guards.py b/tests/unit/test_incompatible_element_guards.py
index cefef1f41..9fa18dd16 100644
--- a/tests/unit/test_incompatible_element_guards.py
+++ b/tests/unit/test_incompatible_element_guards.py
@@ -161,6 +161,39 @@ async def test_input_clear_classifies_incompatible_element(monkeypatch: pytest.M
await element.input_clear()
+def _make_non_stale_locator() -> MagicMock:
+ locator = MagicMock()
+ locator.count = AsyncMock(return_value=1) # refresh_locator_if_stale sees a unique match
+ return locator
+
+
+async def test_input_sequentially_classifies_incompatible_element(monkeypatch: pytest.MonkeyPatch) -> None:
+ # input_sequentially's underlying strategy_aware_input can call locator.fill() directly (the
+ # long-text "fast prefix" path); until this fix that raw Playwright error skipped the same
+ # classification input_fill/input_clear already apply and escaped unclassified (SKY-15219).
+ element = _make_element("huk-input-field", _make_non_stale_locator())
+
+ async def _raise(*args: object, **kwargs: object) -> None:
+ raise PlaywrightError(_NOT_INPUT_ERROR)
+
+ monkeypatch.setattr("skyvern.webeye.utils.dom.handler_utils.input_sequentially", _raise)
+ with pytest.raises(InvalidElementForTextInput):
+ await element.input_sequentially("hello")
+
+
+async def test_input_sequentially_reraises_unrelated_errors(monkeypatch: pytest.MonkeyPatch) -> None:
+ element = _make_element("input", _make_non_stale_locator())
+
+ async def _raise(*args: object, **kwargs: object) -> None:
+ raise PlaywrightError(_TIMEOUT_ERROR)
+
+ monkeypatch.setattr("skyvern.webeye.utils.dom.handler_utils.input_sequentially", _raise)
+ with pytest.raises(PlaywrightError):
+ await element.input_sequentially("hello")
+
+
# --------------------------------------------------------------------------- #
# _strict_date_mask_order / _canonical_iso_date — deterministic date canonicalization.
# A live <input type=date> takes only YYYY-MM-DD; a locale value is canonicalized only when the
diff --git a/tests/unit/test_mcp_argument_validation.py b/tests/unit/test_mcp_argument_validation.py
index ca1b81e43..8b44ee2f6 100644
--- a/tests/unit/test_mcp_argument_validation.py
+++ b/tests/unit/test_mcp_argument_validation.py
@@ -212,3 +212,34 @@ async def test_type_repair_preserves_structured_missing_key_rejection() -> None:
assert error["code"] == "INVALID_INPUT"
assert error["details"]["unsupported_arguments"] == []
assert error["details"]["missing_required_arguments"] == ["workflow_id"]
+
+
+ ("arguments", "expected_field"),
+ [
+ # SKY-15319: page below the tool's documented 1-based minimum.
+ ({"workflow_id": "wpid_test", "page": 0}, "page"),
+ # SKY-14997: a single status value sent as a bare string instead of a list. Not
+ # comma-separated, so the narrow CSV repair correctly leaves it for validation.
+ ({"workflow_id": "wpid_test", "status": "completed"}, "status"),
+ ],
+)
+async def test_out_of_schema_value_gets_structured_error_instead_of_raising(
+ monkeypatch: pytest.MonkeyPatch,
+ arguments: dict,
+ expected_field: str,
+) -> None:
+ list_runs = AsyncMock(return_value=[])
+ monkeypatch.setattr(mcp_workflow, "list_workflow_runs_raw", list_runs)
+
+ # Regression proof: before the fix, FastMCP's own pydantic validation raised
+ # fastmcp.exceptions.ValidationError out of call_next, uncaught by this middleware.
+ payload = await _call("skyvern_workflow_run_list", arguments)
+
+ assert payload["ok"] is False
+ error = payload["error"]
+ assert error["code"] == "INVALID_INPUT"
+ assert expected_field in error["message"]
+ assert "skyvern_workflow_run_list" in error["hint"]
+ list_runs.assert_not_awaited()