Skip to content

Automate browser based workflows with AI

Assert the radio click gate agrees with the driver instead of naming one Playwright version's aria-disabled shapes (SKY-18878) (#9035)

Aaron Perez7350bb5185ca
diff --git a/skyvern/cli/mcp_tools/argument_validation.py b/skyvern/cli/mcp_tools/argument_validation.py
index 74a668031..b6f1bd19a 100644
--- a/skyvern/cli/mcp_tools/argument_validation.py
+++ b/skyvern/cli/mcp_tools/argument_validation.py
@@ -106,32 +106,59 @@ def _millisecond_bounds(schema: Any, *, minimum_floor: float) -> tuple[float, fl
return minimum, maximum
-def _rescale_seconds_to_milliseconds(name: str, schema: Any, value: Any) -> Any:
- """Scale a sub-floor value on a millisecond parameter, which callers undershoot in seconds.
-
- The name gate matters as much as the bound: a non-duration parameter could legitimately
- carry a 1000 minimum, and multiplying its value by 1000 would be silent corruption.
- """
+def _millisecond_argument_bounds(name: str, schema: Any) -> tuple[float, float | None] | None:
if name == "timeout":
minimum_floor = _MILLISECOND_FLOOR
elif name.endswith("_ms"):
- # An explicit unit suffix identifies milliseconds even when the schema floor is < 1s.
minimum_floor = 0
else:
- return value
+ return None
+ return _millisecond_bounds(schema, minimum_floor=minimum_floor)
+
+
+def _millisecond_input_error(name: str, schema: Any, value: Any) -> tuple[str, str] | None:
+ if not isinstance(value, (int, float)) or isinstance(value, bool):
+ return None
+ bounds = _millisecond_argument_bounds(name, schema)
+ if bounds is None:
+ return None
+ minimum, maximum = bounds
+ if not 0 < value < minimum:
+ return None
+ rescaled = value * 1000
+ if rescaled >= minimum and (maximum is None or rescaled <= maximum):
+ return None
+ if maximum is None:
+ return (
+ f"{name} is in milliseconds (minimum {minimum:g}); got {value:g} - "
+ f"for seconds, use at least {minimum / 1000:g}",
+ f"Provide at least {minimum:g} milliseconds or {minimum / 1000:g} seconds for {name}.",
+ )
+ seconds_minimum, seconds_maximum = minimum / 1000, maximum / 1000
+ return (
+ f"{name} is in milliseconds ({minimum:g}-{maximum:g}); got {value:g} - "
+ f"for seconds, use {seconds_minimum:g}-{seconds_maximum:g}",
+ f"For seconds, pass {seconds_minimum:g}-{seconds_maximum:g}; otherwise provide milliseconds from "
+ f"{minimum:g}-{maximum:g}.",
+ )
+
+
+def _rescale_seconds_to_milliseconds(name: str, schema: Any, value: Any) -> Any:
+ """Scale a sub-floor value only when it is valid as seconds and milliseconds.
+
+ The name gate matters as much as the bound: a non-duration parameter could legitimately
+ carry a 1000 minimum, and multiplying its value by 1000 would be silent corruption.
+ """
if not isinstance(value, (int, float)) or isinstance(value, bool):
return value
- bounds = _millisecond_bounds(schema, minimum_floor=minimum_floor)
+ bounds = _millisecond_argument_bounds(name, schema)
if bounds is None:
return value
minimum, maximum = bounds
if not 0 < value < minimum:
return value
rescaled = value * 1000
- if rescaled < minimum:
- return value
- # Rescaling past the ceiling would swap one validation failure for a less recognizable one.
- if maximum is not None and rescaled > maximum:
+ if rescaled < minimum or (maximum is not None and rescaled > maximum):
return value
return int(rescaled) if isinstance(rescaled, float) and rescaled.is_integer() else rescaled
@@ -146,16 +173,16 @@ def _encode_json_object(value: Any) -> Any:
return encoded if len(encoded) <= _MAX_REPAIR_STRING_LENGTH else value
-def _repair_argument_types(tool_name: str, tool: Any, arguments: dict[str, Any]) -> None:
+def _repair_argument_types(tool_name: str, tool: Any, arguments: dict[str, Any]) -> tuple[str, str] | None:
if tool_name == "skyvern_browser_session_create" and arguments.get("generate_browser_profile") is None:
arguments.pop("generate_browser_profile", None)
parameters = getattr(tool, "parameters", None)
if not isinstance(parameters, dict):
- return
+ return None
properties = parameters.get("properties")
if not isinstance(properties, dict):
- return
+ return None
for name, value in arguments.items():
schema = properties.get(name)
@@ -168,7 +195,11 @@ def _repair_argument_types(tool_name: str, tool: Any, arguments: dict[str, Any])
):
arguments[name] = _encode_json_object(value)
else:
+ unit_error = _millisecond_input_error(name, schema, value)
+ if unit_error is not None:
+ return unit_error
arguments[name] = _rescale_seconds_to_milliseconds(name, schema, value)
+ return None
def _argument_contract(tool: Any) -> tuple[set[str], set[str]] | None:
@@ -232,10 +263,12 @@ class MCPArgumentValidationMiddleware(Middleware):
repaired = dict(arguments)
try:
- _repair_argument_types(tool_name, tool, repaired)
+ unit_error = _repair_argument_types(tool_name, tool, repaired)
except Exception:
LOG.warning("mcp_argument_type_repair_failed", tool=tool_name, exc_info=True)
else:
+ if unit_error is not None:
+ return _invalid_input_result(tool_name, *unit_error)
if repaired != arguments:
# FastMCP dispatch reads this same dict reference, so rebinding `arguments` would discard the repair.
arguments.clear()
@@ -301,21 +334,25 @@ def _rejection_result(
message = f"{tool_name} was called with {'; '.join(problems)}."
expected_str = ", ".join(expected) if expected else "(none)"
+ return _invalid_input_result(
+ tool_name,
+ message,
+ f"Call {tool_name} using only its documented parameters: {expected_str}.",
+ details={
+ "unsupported_argument_count" if tool_name in _VALUE_SAFE_TOOLS else "unsupported_arguments": (
+ len(unsupported) if tool_name in _VALUE_SAFE_TOOLS else unsupported
+ ),
+ "missing_required_arguments": missing,
+ "expected_arguments": expected,
+ },
+ )
+
+
+def _invalid_input_result(tool_name: str, message: str, hint: str, details: dict[str, Any] | None = None) -> ToolResult:
payload = make_result(
tool_name,
ok=False,
- error=make_error(
- ErrorCode.INVALID_INPUT,
- message,
- f"Call {tool_name} using only its documented parameters: {expected_str}.",
- details={
- "unsupported_argument_count" if tool_name in _VALUE_SAFE_TOOLS else "unsupported_arguments": (
- len(unsupported) if tool_name in _VALUE_SAFE_TOOLS else unsupported
- ),
- "missing_required_arguments": missing,
- "expected_arguments": expected,
- },
- ),
+ error=make_error(ErrorCode.INVALID_INPUT, message, hint, details=details),
)
text = json.dumps(payload, ensure_ascii=False, default=str)
return ToolResult(content=[TextContent(type="text", text=text)], structured_content=payload)
diff --git a/skyvern/forge/taskv3/tools.py b/skyvern/forge/taskv3/tools.py
index ff3fd230d..2800b8994 100644
--- a/skyvern/forge/taskv3/tools.py
+++ b/skyvern/forge/taskv3/tools.py
@@ -15825,8 +15825,8 @@ def build_browser_tools(
skin_probe = await _probe_evaluate(page, _SKINNED_CHECKBOX_PROBE_JS, selector, pre_click_arg)
except Exception:
skin_probe = None
- # Every toggle answer carries `radio`. The browser engine's own enabled check decides (its aria-disabled
- # rule moves between driver versions), so the label click below is never looser than a plain click.
+ # Every toggle answer carries `radio`. The browser engine's own enabled check decides, whatever its
+ # aria-disabled rule is on this driver, so the label click below is never looser than a plain click.
if not (isinstance(skin_probe, dict) and "radio" in skin_probe):
break
if await _engine_enabled(page, selector):
diff --git a/tests/unit/test_mcp_argument_validation.py b/tests/unit/test_mcp_argument_validation.py
index 76fbed8eb..51b07f0e6 100644
--- a/tests/unit/test_mcp_argument_validation.py
+++ b/tests/unit/test_mcp_argument_validation.py
@@ -272,6 +272,38 @@ async def test_composite_navigate_sub_1000_timeout_reaches_the_navigate_step_as_
assert operations["navigate"]["timeout"] == 45000
+ ("timeout", "expected_timeout", "expected_error"),
+ [
+ (60, 60000, None),
+ (90, None, "timeout is in milliseconds (1000-60000); got 90 - for seconds, use 1-60"),
+ (100, None, "timeout is in milliseconds (1000-60000); got 100 - for seconds, use 1-60"),
+ (500, None, "timeout is in milliseconds (1000-60000); got 500 - for seconds, use 1-60"),
+ ],
+)
+async def test_click_timeout_repairs_valid_seconds_and_rejects_ambiguous_values_at_mcp_dispatch(
+ monkeypatch: pytest.MonkeyPatch, timeout: int, expected_timeout: int | None, expected_error: str | None
+) -> None:
+ click = AsyncMock(return_value="#submit")
+ page = make_skyvern_page(make_mock_page())
+ page.click = click
+ context = BrowserContext(mode="cloud_session", session_id="pbs_test")
+ monkeypatch.setattr(mcp_browser, "get_page", AsyncMock(return_value=(page, context)))
+
+ payload = await _call("skyvern_click", {"selector": "#submit", "selector_mode": "direct", "timeout": timeout})
+
+ if expected_error is not None:
+ assert payload["ok"] is False
+ assert payload["error"]["code"] == "INVALID_INPUT"
+ assert payload["error"]["message"] == expected_error
+ assert payload["error"]["hint"] == "For seconds, pass 1-60; otherwise provide milliseconds from 1000-60000."
+ click.assert_not_awaited()
+ else:
+ assert payload["ok"] is True
+ assert _awaited_kwargs(click)["timeout"] == expected_timeout
+
+
@pytest.mark.asyncio
@pytest.mark.parametrize(
("tool_name", "argument", "sent", "expected"),
@@ -288,8 +320,9 @@ async def test_composite_navigate_sub_1000_timeout_reaches_the_navigate_step_as_
("skyvern_run_task", "timeout_seconds", 45, 45),
("skyvern_workflow_run", "timeout_seconds", 30, 30),
("skyvern_login", "timeout_seconds", 60, 60),
- # Rescaling past the published ceiling would trade one validation failure for another.
- ("skyvern_click", "timeout", 90, 90),
+ # Seconds are converted only when the result fits the schema's millisecond range.
+ ("skyvern_click", "timeout", 45, 45000),
+ ("skyvern_click", "timeout", 60, 60000),
],
)
async def test_timeout_repair_rescales_millisecond_parameters_only(
diff --git a/tests/unit/test_taskv3_tools.py b/tests/unit/test_taskv3_tools.py
index 051af3b91..625af3425 100644
--- a/tests/unit/test_taskv3_tools.py
+++ b/tests/unit/test_taskv3_tools.py
@@ -748,8 +748,8 @@ async def test_click_refuses_a_natively_disabled_target_before_the_actionability
blinked = await click({"selector": "#blink-on"})
# The gate stands in for the driver's own enabled check, so the contract is agreement with it,
# asserted against `is_enabled` rather than against a hardcoded rule. Which shapes the driver
- # refuses is version-dependent (`aria-disabled` on an ANCESTOR is refused by 1.61 and clicked
- # by 1.58), and a test naming the shapes pins the gate to whichever driver it was written on.
+ # refuses is version-dependent (`aria-disabled` on a role-less ANCESTOR is clicked by 1.46 and
+ # refused by 1.56), and a test naming the shapes pins the gate to the driver it was written on.
aria: dict[str, tuple[bool, str | None, str, float]] = {}
for selector in ("#aria-off", "#aria-under-wrapper", "#declared-off"):
driver_refuses = not await page.locator(selector).is_enabled()
@@ -17549,7 +17549,7 @@ async def test_type_says_an_aria_disabled_field_is_disabled_instead_of_waiting_f
# fill() waits on the driver's own rule, which is wider than `:disabled`, so a gate reading
# `:disabled` alone pays the full timeout on a field the driver was never going to fill. Asserted
# as agreement with `is_enabled` rather than per shape: which shapes it refuses moves between
- # driver versions (`aria-disabled` on an ANCESTOR is refused by 1.61 and filled by 1.58).
+ # driver versions (`aria-disabled` on a role-less ANCESTOR is filled by 1.46 and refused by 1.56).
html = (
'<input id="own" type="text" aria-disabled="true" style="width:200px;height:30px">'
'<div aria-disabled="true"><input id="under" type="text" style="width:200px;height:30px"></div>'