Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
SKY-15160: Preserve failed scout interaction evidence (#8325)
Andrew Neilson76d4589b918a
diff --git a/skyvern/forge/sdk/copilot/tools/mcp_hooks.py b/skyvern/forge/sdk/copilot/tools/mcp_hooks.pyindex aa719609a..7ca26653f 100644--- a/skyvern/forge/sdk/copilot/tools/mcp_hooks.py+++ b/skyvern/forge/sdk/copilot/tools/mcp_hooks.py@@ -29,11 +29,16 @@ from skyvern.forge.sdk.copilot.mcp_adapter import (BROWSER_TARGET_PARAM,BROWSER_TARGET_PARAM_NAME,SchemaOverlay,+ _scrub_tool_result,)from skyvern.forge.sdk.copilot.output_extraction_plan import (requested_output_designation_capability,unbound_candidate_relations,)+from skyvern.forge.sdk.copilot.output_utils import (+ mark_mcp_result_untrusted_for_llm,+ sanitize_tool_result_for_llm,+)from skyvern.forge.sdk.copilot.page_identity import safe_page_originfrom skyvern.forge.sdk.copilot.reached_download_target import download_claim_helper_contractfrom skyvern.forge.sdk.copilot.request_policy import (@@ -85,6 +90,7 @@ from .page_observation import (_resolve_url_title,)from .scouting import (+ _SCOUT_RESULT_CHAR_CAP,_arm_scout_download_listener,_arm_scout_popup_listener,_attach_evaluate_page_facts,@@ -99,6 +105,7 @@ from .scouting import (_mark_pending_browser_interaction_observation,_maybe_attach_observed_download_target,_maybe_attach_observed_render_target,+ _page_evidence_location_fingerprint,_page_evidence_names_obstruction,_prenav_ambiguity_for_selector,_prenav_role_name_for_selector,@@ -106,7 +113,9 @@ from .scouting import (_record_scouted_interaction,_register_scout_interaction_observation,_resolve_scout_role_name,+ _scout_act_observe_page_evidence,_scout_session_download_names,+ _shed_scout_page_summary_section,)LOG = structlog.get_logger()@@ -190,6 +199,120 @@ def _effective_target_text(selector: str, role: str = "", accessible_name: str =return label or selector+def _failed_click_attempted_control(+ *,+ selector: str,+ selector_candidates: list[ScoutedSelectorCandidate] | None,+ selector_match_count: int | None,+ role: str,+ accessible_name: str,+ role_name_match_count: int | None,+ ambiguous: bool,+) -> dict[str, Any]:+ control: dict[str, Any] = {+ "selector": selector,+ "effective_target": _effective_target_text(selector, role, accessible_name),+ }+ if selector_candidates:+ control["selector_candidates"] = selector_candidates+ if selector_match_count is not None:+ control["selector_match_count"] = selector_match_count+ if role:+ control["role"] = role+ if accessible_name:+ control["accessible_name"] = accessible_name+ if role_name_match_count is not None:+ control["role_name_match_count"] = role_name_match_count+ if ambiguous:+ control["ambiguous"] = True+ return control+++_FAILED_CLICK_ENRICHMENT_TEXT_MAX_CHARS = 160+_FAILED_CLICK_TRUNCATION_SUFFIX = "... [truncated]"+++def _bound_failed_click_result(ctx: AgentContext, result: dict[str, Any]) -> None:+ """Bound the complete model-visible failure while retaining typed control/error facts."""++ scrubbed = _scrub_tool_result(ctx, result)+ if isinstance(scrubbed, dict) and scrubbed is not result:+ result.clear()+ result.update(scrubbed)++ def model_visible_size() -> int:+ sanitized = sanitize_tool_result_for_llm("click", result)+ return len(json.dumps(mark_mcp_result_untrusted_for_llm(sanitized)))++ def bound_enrichment_text(+ container: dict[str, Any],+ key: str,+ max_chars: int = _FAILED_CLICK_ENRICHMENT_TEXT_MAX_CHARS,+ ) -> None:+ value = container.get(key)+ if not isinstance(value, str) or len(value) <= max_chars:+ return+ prefix_chars = max_chars - len(_FAILED_CLICK_TRUNCATION_SUFFIX)+ container[key] = value[:prefix_chars] + _FAILED_CLICK_TRUNCATION_SUFFIX++ data = result.get("data")+ if not isinstance(data, dict):+ return+ attempted_control = data.get("attempted_control")+ if not isinstance(attempted_control, dict):+ return+ if model_visible_size() <= _SCOUT_RESULT_CHAR_CAP:+ return++ # Page facts are optional enrichment. Shed their detail before reducing the attempted-control+ # packet or typed failure, including the minimal shed marker left by the summary builder.+ page = data.get("page")+ if isinstance(page, dict):+ raw_shed = page.get("shed")+ shed = [value for value in raw_shed if isinstance(value, str)] if isinstance(raw_shed, list) else []+ while model_visible_size() > _SCOUT_RESULT_CHAR_CAP:+ section = _shed_scout_page_summary_section(page)+ if section is None:+ data.pop("page", None)+ break+ shed.append(section)+ page["shed"] = shed+ result.pop("warnings", None)+ for key in (+ "selector_candidates",+ "role_name_match_count",+ "selector_match_count",+ "ambiguous",+ "accessible_name",+ "role",+ ):+ if model_visible_size() <= _SCOUT_RESULT_CHAR_CAP:+ return+ attempted_control.pop(key, None)++ # The typed failure is the tool's authoritative result and must remain unchanged. Compact only+ # enriched control/location detail, retaining stable prefixes and explicit truncation markers.+ compactable_enrichment_fields = (+ (attempted_control, "selector"),+ (attempted_control, "effective_target"),+ (data, "url"),+ )+ for container, key in compactable_enrichment_fields:+ bound_enrichment_text(container, key)++ minimum_chars = len(_FAILED_CLICK_TRUNCATION_SUFFIX) + 1+ while model_visible_size() > _SCOUT_RESULT_CHAR_CAP:+ remaining = [+ (container, key, value)+ for container, key in compactable_enrichment_fields+ if isinstance((value := container.get(key)), str) and len(value) > minimum_chars+ ]+ if not remaining:+ break+ container, key, value = max(remaining, key=lambda item: len(item[2]))+ bound_enrichment_text(container, key, max(minimum_chars, len(value) // 2))++async def _get_block_schema_pre_hook(params: dict[str, Any],ctx: AgentContext,@@ -830,6 +953,7 @@ async def _click_post_hook(pending_selector_candidates = getattr(ctx, "pending_scout_selector_candidates", None)ctx.pending_scout_selector_candidates = Nonectx.pending_scout_reanchor = None+ pending_click_selector = ctx.pending_scout_click_selectorctx.pending_scout_click_selector = Noneif result.get("ok") and result.get("data"):data = result["data"]@@ -905,6 +1029,46 @@ async def _click_post_hook("The page observation did not change after the click; no post-click page evidence was attached."),}+ elif not result.get("ok") and isinstance(pending_click_selector, str) and pending_click_selector.strip():+ selector = pending_click_selector.strip()+ role, accessible_name = _prenav_role_name_for_selector(pending_role_name, selector)+ selector_match_count = (+ pending_selector_match_count[1]+ if isinstance(pending_selector_match_count, tuple)+ and len(pending_selector_match_count) == 2+ and pending_selector_match_count[0] == selector+ else None+ )+ role_name_match_count = (+ pending_role_name_match_count[3]+ if isinstance(pending_role_name_match_count, tuple)+ and len(pending_role_name_match_count) == 4+ and pending_role_name_match_count[:3] == (selector, role, accessible_name)+ else None+ )+ attempted_control = _failed_click_attempted_control(+ selector=selector,+ selector_candidates=pending_selector_candidates,+ selector_match_count=selector_match_count,+ role=role,+ accessible_name=accessible_name,+ role_name_match_count=role_name_match_count,+ ambiguous=_prenav_ambiguity_for_selector(pending_ambiguous, selector),+ )+ url, _ = await _resolve_url_title(raw, ctx)+ result["data"] = {+ "attempted_control": attempted_control,+ "url": safe_page_origin(url) or "",+ }+ location_fingerprint = _page_evidence_location_fingerprint(url)+ if location_fingerprint is not None:+ result["data"]["current_url_location_fingerprint"] = location_fingerprint+ captured_url = url or None+ if url:+ page_evidence = await _scout_act_observe_page_evidence(ctx, url=url)+ if page_evidence is not None:+ _attach_scout_page_summary(ctx, result, page_evidence)+ _bound_failed_click_result(ctx, result)# The round-trip is skipped only when the evidence positively names the obstruction a frame# would have shown; evidence that merely parsed is not a substitute for looking at the page.if ctx.last_scout_act_observe_outcome != "attached" or not _page_evidence_names_obstruction(page_evidence):diff --git a/tests/unit/test_copilot_scout_act_observe.py b/tests/unit/test_copilot_scout_act_observe.pyindex 5ac318a1b..4dd52d675 100644--- a/tests/unit/test_copilot_scout_act_observe.py+++ b/tests/unit/test_copilot_scout_act_observe.py@@ -43,7 +43,10 @@ from skyvern.forge.sdk.copilot.context import (from skyvern.forge.sdk.copilot.enforcement import (record_scouted_output_coverage,)+from skyvern.forge.sdk.copilot.mcp_adapter import SchemaOverlay, SkyvernOverlayMCPServerfrom skyvern.forge.sdk.copilot.output_extraction_plan import ShapeExpectation, ValueCardinality, ValueShape+from skyvern.forge.sdk.copilot.output_utils import MCP_RESULT_PROVENANCE_KEY, MCP_RESULT_PROVENANCE_VALUE+from skyvern.forge.sdk.copilot.page_identity import safe_page_originfrom skyvern.forge.sdk.copilot.request_policy import CompletionCriterionfrom skyvern.forge.sdk.copilot.result_evidence import scout_observation_bound_pathsfrom skyvern.forge.sdk.copilot.runtime import AgentContext, bound_call_browser_session@@ -61,7 +64,7 @@ from skyvern.forge.sdk.copilot.tools.scouting import (_safe_page_evidence_url,_scout_act_observe_page_evidence,)-from tests.unit.copilot_test_helpers import carried_interaction+from tests.unit.copilot_test_helpers import carried_interaction, make_copilot_ctx_SOURCE_URL = "https://example.com/product"_LANDING_URL = "https://example.com/results"@@ -234,6 +237,356 @@ def _flow_by_step(ctx: SimpleNamespace) -> dict[int, tuple[dict[str, Any], str]]return {entry["step"]: (entry["evidence"], entry["reached_via"]) for entry in ctx.flow_evidence}+async def _failed_click_through_wrapper(+ monkeypatch: pytest.MonkeyPatch,+ *,+ resulting_url: str,+ page_payload: dict[str, Any] | None,+ source_url: str = _SOURCE_URL,+ browser_title: str = "Current options",+ selector: str = "#continue",+ accessible_name: str = "Continue",+ error_message: str = "element not interactable",+ error_hint: str = "Target remained covered",+ selector_candidate_count: int = 1,+ registered_secrets: list[str] | None = None,+ codeblock_redaction_parameters: dict[str, Any] | None = None,+) -> tuple[dict[str, Any], Any, AgentContext]:+ async def call_internal_tool(_tool: str, arguments: dict[str, Any]) -> dict[str, Any]:+ if "REQUESTED_TARGETS" in str(arguments.get("expression", "")):+ if page_payload is None:+ return {"ok": False, "error": "page evidence unavailable"}+ return {"ok": True, "data": {"result": page_payload}}+ return {+ "ok": True,+ "data": {+ "result": {+ "role_name": {"role": "button", "accessible_name": accessible_name},+ "role_name_match_count": 1,+ "selector_match_count": 1,+ "selector_candidates": [+ {+ "selector": selector+ if selector_candidate_count == 1+ else f"{selector}-{index}-{'x' * 200}",+ "source": "id",+ "match_count": 1,+ }+ for index in range(selector_candidate_count)+ ],+ }+ },+ }++ ctx = make_copilot_ctx()+ ctx.secret_scrub_values = registered_secrets or []+ ctx.codeblock_redaction_parameters = codeblock_redaction_parameters or {}+ ctx.discovery_mcp_server = SimpleNamespace(call_internal_tool=AsyncMock(side_effect=call_internal_tool))+ monkeypatch.setattr(scouting_module, "_live_working_page_url", AsyncMock(return_value=source_url))++ raw_result = SimpleNamespace(+ structured_content={+ "ok": False,+ "error": {+ "code": "ELEMENT_NOT_INTERACTABLE",+ "message": error_message,+ "hint": error_hint,+ },+ "browser_context": {"url": resulting_url, "title": browser_title},+ },+ is_error=True,+ content=[],+ )+ server = SkyvernOverlayMCPServer(+ transport=MagicMock(),+ overlays={"click": SchemaOverlay(pre_hook=tools_module._click_pre_hook, post_hook=_click_post_hook)},+ alias_map={},+ allowlist=frozenset(),+ context_provider=lambda: ctx,+ )+ server._client = SimpleNamespace(call_tool=AsyncMock(return_value=raw_result))++ wrapped = await server.call_tool("click", {"selector": selector})+ return json.loads(wrapped.content[0].text), wrapped, ctx+++def _failed_page_payload() -> dict[str, Any]:+ return {+ "page_title": "Current options",+ "forms": [+ {+ "fields": [],+ "submit_controls": [+ {+ "text": "Try another option",+ "type": "button",+ "selector": "#try-another",+ "selector_candidates": [{"selector": "#try-another", "source": "id", "match_count": 1}],+ }+ ],+ }+ ],+ "navigation_targets": [],+ "result_containers": [],+ }+++ "resulting_url",+ [+ _SOURCE_URL,+ "https://example.net/wrong-target?token=not-model-visible",+ ],+)+async def test_failed_click_wrapper_carries_attempt_and_current_page_without_success_credit(+ monkeypatch: pytest.MonkeyPatch,+ resulting_url: str,+) -> None:+ projected, wrapped, ctx = await _failed_click_through_wrapper(+ monkeypatch,+ resulting_url=resulting_url,+ page_payload=_failed_page_payload(),+ )++ assert wrapped.isError is True+ assert projected["ok"] is False+ assert projected["error"] == "element not interactable. Target remained covered"+ assert projected["error_code"] == "ELEMENT_NOT_INTERACTABLE"+ assert projected[MCP_RESULT_PROVENANCE_KEY] == MCP_RESULT_PROVENANCE_VALUE+ assert projected["data"]["attempted_control"] == {+ "selector": "#continue",+ "effective_target": "button Continue",+ "selector_candidates": [{"selector": "#continue", "source": "id", "match_count": 1}],+ "selector_match_count": 1,+ "role": "button",+ "accessible_name": "Continue",+ "role_name_match_count": 1,+ }+ assert projected["data"]["url"] == safe_page_origin(resulting_url)+ assert projected["data"]["current_url_location_fingerprint"] == _page_evidence_location_fingerprint(resulting_url)+ assert "title" not in projected["data"]+ assert projected["data"]["page"]["forms"][0]["submit_controls"][0]["text"] == "Try another option"+ assert "not-model-visible" not in json.dumps(projected)+ assert len(json.dumps(projected)) <= _SCOUT_RESULT_CHAR_CAP+ assert ctx.scouted_interactions == []+ assert ctx.scout_trajectory == []+ assert ctx.flow_evidence == []+++async def test_failed_click_page_evidence_ablation_preserves_attempt_and_failure(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ with_page, _, _ = await _failed_click_through_wrapper(+ monkeypatch,+ resulting_url=_SOURCE_URL,+ page_payload=_failed_page_payload(),+ )+ without_page, _, _ = await _failed_click_through_wrapper(+ monkeypatch,+ resulting_url=_SOURCE_URL,+ page_payload=None,+ )++ assert "page" not in without_page["data"]+ assert with_page["data"].pop("page")+ assert with_page == without_page+++async def test_failed_click_without_page_evidence_sheds_oversized_candidates_but_keeps_failure_facts(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ projected, wrapped, _ = await _failed_click_through_wrapper(+ monkeypatch,+ resulting_url=_SOURCE_URL,+ page_payload=None,+ selector_candidate_count=20,+ )++ assert wrapped.isError is True+ assert projected["ok"] is False+ assert projected["error"] == "element not interactable. Target remained covered"+ assert projected["error_code"] == "ELEMENT_NOT_INTERACTABLE"+ assert projected["data"]["attempted_control"]["selector"] == "#continue"+ assert projected["data"]["attempted_control"]["effective_target"] == "button Continue"+ assert "selector_candidates" not in projected["data"]["attempted_control"]+ assert len(json.dumps(projected)) <= _SCOUT_RESULT_CHAR_CAP+++async def test_failed_click_without_page_evidence_preserves_typed_failure_while_bounding_control_detail(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ selector = "#oversized-required-control-" + "s" * (_SCOUT_RESULT_CHAR_CAP * 2)+ accessible_name = "Oversized required control " + "n" * (_SCOUT_RESULT_CHAR_CAP * 2)+ error_message = "oversized typed click failure " + "e" * 500+ projected, wrapped, _ = await _failed_click_through_wrapper(+ monkeypatch,+ resulting_url=_SOURCE_URL,+ page_payload=None,+ selector=selector,+ accessible_name=accessible_name,+ error_message=error_message,+ error_hint="",+ )++ assert wrapped.isError is True+ assert projected["ok"] is False+ assert projected["error"] == error_message+ assert projected["error_code"] == "ELEMENT_NOT_INTERACTABLE"+ assert "#oversized-required-control-" in projected["data"]["attempted_control"]["selector"]+ assert "Oversized required control" in projected["data"]["attempted_control"]["effective_target"]+ assert len(json.dumps(projected)) <= _SCOUT_RESULT_CHAR_CAP+++async def test_failed_click_wrapper_reuses_registered_and_codeblock_redaction_boundaries(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ credential_secret = "registered-password-5ac091"+ parameter_secret = "codeblock-parameter-26bc18"+ location_value = "location-only-value-7fe2b7"+ page_payload = _failed_page_payload()+ page_payload["page_title"] = f"Account for {credential_secret}"+ page_payload["forms"][0]["submit_controls"][0]["text"] = f"Continue with {credential_secret} and {parameter_secret}"++ def redact_codeblock_parameters(value: Any, parameters: dict[str, Any]) -> Any:+ assert parameters == {"account": parameter_secret}+ return value.replace(parameter_secret, "[REDACTED_PARAMETER]") if isinstance(value, str) else value++ codeblock_scrubber = MagicMock(side_effect=redact_codeblock_parameters)+ monkeypatch.setattr(tools_module.app.AGENT_FUNCTION, "redact_codeblock_parameter_values", codeblock_scrubber)++ projected, wrapped, ctx = await _failed_click_through_wrapper(+ monkeypatch,+ resulting_url=f"https://example.com/checkout?state={location_value}",+ page_payload=page_payload,+ registered_secrets=[credential_secret],+ codeblock_redaction_parameters={"account": parameter_secret},+ )++ assert wrapped.isError is True+ assert projected["ok"] is False+ assert projected["error_code"] == "ELEMENT_NOT_INTERACTABLE"+ assert projected["data"]["attempted_control"]["selector"] == "#continue"+ serialized = json.dumps(projected)+ assert projected["data"]["url"] == "https://example.com/"+ assert projected["data"]["current_url_location_fingerprint"] == _page_evidence_location_fingerprint(+ f"https://example.com/checkout?state={location_value}"+ )+ assert credential_secret not in serialized+ assert parameter_secret not in serialized+ assert location_value not in serialized+ assert projected["data"]["page"]["page_title"] == "Account for [REDACTED_SECRET]"+ assert (+ projected["data"]["page"]["forms"][0]["submit_controls"][0]["text"]+ == "Continue with [REDACTED_SECRET] and [REDACTED_PARAMETER]"+ )+ assert codeblock_scrubber.called+ assert len(serialized) <= _SCOUT_RESULT_CHAR_CAP+ assert ctx.scouted_interactions == []+ assert ctx.scout_trajectory == []+ assert ctx.flow_evidence == []+++async def test_failed_click_preserves_ordinary_page_facts_matching_short_url_components(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ resulting_url = "https://example.net/go?state=on#tab"+ page_payload = _failed_page_payload()+ page_payload["page_title"] = "Go on"+ page_payload["forms"][0]["submit_controls"][0]["text"] = "Open tab"++ projected, wrapped, ctx = await _failed_click_through_wrapper(+ monkeypatch,+ resulting_url=resulting_url,+ page_payload=page_payload,+ )++ assert wrapped.isError is True+ assert projected["ok"] is False+ assert projected["error_code"] == "ELEMENT_NOT_INTERACTABLE"+ assert projected["data"]["attempted_control"]["selector"] == "#continue"+ assert projected["data"]["url"] == "https://example.net/"+ assert projected["data"]["current_url_location_fingerprint"] == _page_evidence_location_fingerprint(resulting_url)+ serialized = json.dumps(projected)+ assert projected["data"]["page"]["page_title"] == "Go on"+ assert projected["data"]["page"]["forms"][0]["submit_controls"][0]["text"] == "Open tab"+ assert len(serialized) <= _SCOUT_RESULT_CHAR_CAP+ assert ctx.scouted_interactions == []+ assert ctx.scout_trajectory == []+ assert ctx.flow_evidence == []+++async def test_failed_click_cap_is_enforced_after_registered_short_secret_scrub(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ registered_secret = "654321"+ reflected_secret = " ".join([registered_secret] * 20)+ page_payload = _failed_page_payload()+ page_payload["forms"] = [+ {+ "fields": [],+ "submit_controls": [+ {"text": reflected_secret, "type": "button", "selector": f"#choice-{form_index}-{index}"}+ for index in range(4)+ ],+ }+ for form_index in range(4)+ ]++ projected, wrapped, ctx = await _failed_click_through_wrapper(+ monkeypatch,+ resulting_url=_SOURCE_URL,+ page_payload=page_payload,+ registered_secrets=[registered_secret],+ )++ serialized = json.dumps(projected)+ assert wrapped.isError is True+ assert projected["ok"] is False+ assert projected["error_code"] == "ELEMENT_NOT_INTERACTABLE"+ assert projected["data"]["attempted_control"]["selector"] == "#continue"+ assert projected["data"]["attempted_control"]["effective_target"] == "button Continue"+ assert projected["data"]["url"] == safe_page_origin(_SOURCE_URL)+ assert projected["data"]["current_url_location_fingerprint"] == _page_evidence_location_fingerprint(_SOURCE_URL)+ assert registered_secret not in serialized+ assert len(serialized) <= _SCOUT_RESULT_CHAR_CAP+ assert ctx.scouted_interactions == []+ assert ctx.scout_trajectory == []+ assert ctx.flow_evidence == []+++async def test_failed_click_resulting_location_ablation_changes_only_safe_location_facts(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ same_page, _, _ = await _failed_click_through_wrapper(+ monkeypatch,+ resulting_url=_SOURCE_URL,+ page_payload=_failed_page_payload(),+ )+ redirected, _, _ = await _failed_click_through_wrapper(+ monkeypatch,+ resulting_url="https://example.net/wrong-target?token=second",+ page_payload=_failed_page_payload(),+ )++ assert same_page["data"]["url"] != redirected["data"]["url"]+ assert (+ same_page["data"]["current_url_location_fingerprint"] != redirected["data"]["current_url_location_fingerprint"]+ )+ assert same_page["data"]["page"]["page_title"] == "Current options"+ assert redirected["data"]["page"]["page_title"] == "Current options"+ same_page["data"]["url"] = redirected["data"]["url"]+ same_page["data"]["current_url_location_fingerprint"] = redirected["data"]["current_url_location_fingerprint"]+ assert same_page == redirected++def test_safe_page_evidence_url_keeps_only_origin_and_fingerprints_location() -> None:assert (_safe_page_evidence_url("https://user:[email protected]/callback?code=secret#access_token=secret")