Skip to content

Automate browser based workflows with AI

SKY-10965 Add Copilot hook log attribution (#6578)

Andrew Neilsonae8fcddb2f4b
diff --git a/skyvern/config.py b/skyvern/config.py
index e14212080..8218dddac 100644
--- a/skyvern/config.py
+++ b/skyvern/config.py
@@ -157,6 +157,12 @@ class Settings(BaseSettings):
# Off restores schema-less interaction packets plus standalone page inspects.
COPILOT_SCOUT_ACT_OBSERVE_ENABLED: bool = True
COPILOT_SCOUT_ACT_OBSERVE_TIMEOUT_SECONDS: float = 4.0
+ # Steer a repeat evaluate on an unchanged page toward acting on an already-identified target.
+ # Off restores the raw evaluate result with no actionable-target naming or act instruction.
+ COPILOT_EVALUATE_ACTIONABLE_TARGET_STEER_ENABLED: bool = True
+ # On a repeat evaluate of an unchanged page with exactly one safe target, click it in-process
+ # instead of only advising. Off (default) keeps the advisory-only steer above.
+ COPILOT_EVALUATE_AUTO_ACT_ON_REPEAT_ENABLED: bool = False
# Persist copilot completion criteria per goal epoch with tri-state verdicts
# and definition-plane grading. Off restores per-turn re-derivation, the legacy
# run-status success claim, and unknown-verdicts-as-unsatisfied consumer behavior.
diff --git a/skyvern/forge/sdk/copilot/hooks.py b/skyvern/forge/sdk/copilot/hooks.py
index 19441d913..6b88ce148 100644
--- a/skyvern/forge/sdk/copilot/hooks.py
+++ b/skyvern/forge/sdk/copilot/hooks.py
@@ -48,6 +48,14 @@ _VERIFIED_GOAL_CONTEXT_ATTRS: frozenset[str] = frozenset(
)
+def _copilot_log_fields(ctx: CopilotContext) -> dict[str, str | None]:
+ return {
+ "workflow_permanent_id": getattr(ctx, "workflow_permanent_id", None),
+ "turn_id": getattr(ctx, "turn_id", None),
+ "workflow_copilot_chat_id": getattr(ctx, "workflow_copilot_chat_id", None),
+ }
+
+
def _tool_completion_satisfies_turn(ctx: CopilotContext, tool_name: str, parsed: Mapping[str, object]) -> bool:
if tool_name not in {"run_blocks_and_collect_debug", "update_and_run_blocks"}:
return False
@@ -90,6 +98,7 @@ class CopilotRunHooks(RunHooksBase):
ok=parsed.get("ok"),
summary=summary,
total_tokens=getattr(self._ctx, "total_tokens_used", None),
+ **_copilot_log_fields(self._ctx),
)
if tool_name == "list_credentials" and parsed.get("ok"):
@@ -124,6 +133,7 @@ class CopilotRunHooks(RunHooksBase):
LOG.warning(
"CopilotRunHooks.on_tool_end recording failed, skipping entry",
tool=getattr(tool, "name", None),
+ **_copilot_log_fields(self._ctx),
exc_info=True,
)
return
@@ -142,6 +152,7 @@ class CopilotRunHooks(RunHooksBase):
workflow_run_id=(parsed.get("data") or {}).get("workflow_run_id")
if isinstance(parsed.get("data"), dict)
else None,
+ **_copilot_log_fields(self._ctx),
)
self._ctx.goal_satisfied_tool_name = tool_name
self._ctx.goal_satisfied_tool_output = dict(parsed)
diff --git a/skyvern/forge/sdk/copilot/runtime.py b/skyvern/forge/sdk/copilot/runtime.py
index 47ed80ced..2bff031d6 100644
--- a/skyvern/forge/sdk/copilot/runtime.py
+++ b/skyvern/forge/sdk/copilot/runtime.py
@@ -247,6 +247,9 @@ class AgentContext:
post_run_page_observation_workflow_run_id: str | None = None
post_run_page_observation_after_failed_test: bool = False
post_run_current_page_inspection_workflow_run_id: str | None = None
+ last_evaluate_actionable_signature: str | None = None
+ last_evaluate_actionable_url: str | None = None
+ last_auto_acted_signature: str | None = None
observed_browser_urls: list[str] = field(default_factory=list)
# Ephemeral within-turn scout captures; not persisted across turns.
scouted_interactions: list[ScoutedInteraction] = field(default_factory=list)
diff --git a/skyvern/forge/sdk/copilot/tools/mcp_hooks.py b/skyvern/forge/sdk/copilot/tools/mcp_hooks.py
index 82bc2f931..3e3647236 100644
--- a/skyvern/forge/sdk/copilot/tools/mcp_hooks.py
+++ b/skyvern/forge/sdk/copilot/tools/mcp_hooks.py
@@ -41,6 +41,7 @@ from .scouting import (
_consume_scout_source_url,
_mark_page_inspected,
_mark_pending_browser_interaction_observation,
+ _maybe_steer_evaluate_to_action,
_record_scouted_interaction,
_register_scout_interaction_observation,
_resolve_scout_role_name,
@@ -499,6 +500,7 @@ async def _evaluate_post_hook(
title=title,
observed_data=result["data"],
)
+ await _maybe_steer_evaluate_to_action(ctx, result, url=url)
return result
diff --git a/skyvern/forge/sdk/copilot/tools/scouting.py b/skyvern/forge/sdk/copilot/tools/scouting.py
index a11c5fa91..ab17d9f25 100644
--- a/skyvern/forge/sdk/copilot/tools/scouting.py
+++ b/skyvern/forge/sdk/copilot/tools/scouting.py
@@ -1,6 +1,7 @@
from __future__ import annotations
import asyncio
+import hashlib
import json
import re
import time
@@ -206,6 +207,7 @@ def _record_scouted_interaction(
# press_key may be page-level, so it is recorded by key even with no selector; other tools require one.
if tool_name != "press_key" and not selector:
return
+ _reset_evaluate_actionable_tracker(ctx)
artifact: ScoutedInteraction = {"tool_name": tool_name}
if selector:
artifact["selector"] = selector
@@ -449,6 +451,331 @@ def _attach_scout_page_summary(result: dict[str, Any], page_evidence: dict[str,
LOG.warning("copilot_scout_act_observe_summary_failed", exc_info=True)
+_EVALUATE_ACTIONABLE_MAX_TARGETS = 4
+
+_EVALUATE_ACTIONABLE_ACT_INSTRUCTION = (
+ "This page already exposes actionable targets; click the intended one rather than re-evaluating."
+)
+
+
+def _reset_evaluate_actionable_tracker(ctx: AgentContext) -> None:
+ ctx.last_evaluate_actionable_signature = None
+ ctx.last_evaluate_actionable_url = None
+
+
+def _actionable_target_identities(evidence: dict[str, Any]) -> list[tuple[str, str]]:
+ identities: list[tuple[str, str]] = []
+
+ def add(control: Any) -> None:
+ if not isinstance(control, dict):
+ return
+ selector = _summary_text(control.get("selector"))
+ text = _summary_text(control.get("text") or control.get("value") or control.get("aria_label"))
+ if selector or text:
+ identities.append((selector, text))
+
+ for form in evidence.get("forms") or []:
+ if not isinstance(form, dict):
+ continue
+ for field_entry in form.get("fields") or []:
+ add(field_entry)
+ for control in form.get("submit_controls") or []:
+ add(control)
+ for target in evidence.get("navigation_targets") or []:
+ add(target)
+ for container in evidence.get("result_containers") or []:
+ add(container)
+ for overlay in evidence.get("modal_overlays") or []:
+ if not isinstance(overlay, dict):
+ continue
+ for control in overlay.get("dismiss_controls") or []:
+ add(control)
+ return identities
+
+
+def _actionable_target_signature(identities: list[tuple[str, str]]) -> str:
+ canonical = json.dumps(sorted(identities), separators=(",", ":"))
+ return hashlib.sha256(canonical.encode("utf-8")).hexdigest()
+
+
+def _actionable_targets_for_result(identities: list[tuple[str, str]]) -> list[dict[str, str]]:
+ targets: list[dict[str, str]] = []
+ for selector, text in identities[:_EVALUATE_ACTIONABLE_MAX_TARGETS]:
+ entry = {key: value for key, value in {"selector": selector, "text": text}.items() if value}
+ if entry:
+ targets.append(entry)
+ return targets
+
+
+# Verbs that imply an irreversible or money-moving side effect — never auto-clicked.
+_AUTO_ACT_HIGH_TIER_VERBS = frozenset(
+ {
+ "pay",
+ "payment",
+ "purchase",
+ "buy",
+ "order",
+ "place order",
+ "checkout",
+ "delete",
+ "remove",
+ "transfer",
+ "send",
+ "submit payment",
+ "confirm payment",
+ "wire",
+ "withdraw",
+ "cancel",
+ }
+)
+
+
+def _auto_act_is_high_tier_label(*labels: Any) -> bool:
+ for label in labels:
+ if not isinstance(label, str):
+ continue
+ normalized = label.strip().lower()
+ if not normalized:
+ continue
+ if any(verb in normalized for verb in _AUTO_ACT_HIGH_TIER_VERBS):
+ return True
+ return False
+
+
+def _auto_act_href_is_navigation(href: Any) -> bool:
+ if not isinstance(href, str):
+ return False
+ candidate = href.strip()
+ if not candidate or candidate.startswith("#"):
+ return False
+ lowered = candidate.lower()
+ if lowered.startswith(("javascript:", "mailto:", "tel:")):
+ return False
+ if lowered.startswith(("http://", "https://")):
+ return True
+ return not lowered.startswith(("data:", "blob:"))
+
+
+def _auto_act_candidate(parsed: dict[str, Any]) -> dict[str, Any] | None:
+ """Return the single unambiguous, low-tier nav link to auto-click, or None.
+
+ Eligible only from navigation_targets (`<a href>` with a real http/https/relative
+ href and non-empty text); form submit_controls, form fields, result containers, and
+ modal dismiss controls are excluded. Form submits are never candidates because the
+ structured-evidence producer cannot reliably distinguish a writing submit from a
+ bare default-submit button, so the whole form-submit class is dropped for safety.
+ Money-moving / destructive verbs are dropped. Exactly one survivor ⇒ act; zero or
+ more than one ⇒ None."""
+ candidates: list[dict[str, Any]] = []
+
+ for target in parsed.get("navigation_targets") or []:
+ if not isinstance(target, dict):
+ continue
+ selector = _summary_text(target.get("selector"))
+ if not selector or target.get("disabled") is True:
+ continue
+ if not _auto_act_href_is_navigation(target.get("href")):
+ continue
+ text = _summary_text(target.get("text"))
+ if not text:
+ continue
+ if _auto_act_is_high_tier_label(text, target.get("name"), target.get("id")):
+ continue
+ candidates.append({"selector": selector, "text": text})
+
+ if len(candidates) != 1:
+ return None
+ return candidates[0]
+
+
+_EVALUATE_STEER_SHED_MARKER = "[omitted on repeat — act on the named target instead of re-reading]"
+
+# Keys the steer must never shed: navigation/identity context plus the steer's own output.
+_EVALUATE_STEER_ESSENTIAL_KEYS = frozenset(
+ {"url", "title", "observation_step", "actionable_targets", "next_action", "next_action_reason"}
+)
+
+# Nested bulky subfields inside an evaluate `result` dict (the raw page payload).
+_EVALUATE_STEER_NESTED_BULKY_KEYS = ("html", "outerHTML", "innerHTML", "body", "bodyText", "text", "buttons")
+
+
+def _serialized_len(value: Any) -> int:
+ try:
+ return len(json.dumps(value, default=str))
+ except Exception:
+ return len(str(value))
+
+
+def _largest_non_essential_data_key(data: dict[str, Any]) -> str | None:
+ candidates = [
+ (key, _serialized_len(value))
+ for key, value in data.items()
+ if key not in _EVALUATE_STEER_ESSENTIAL_KEYS and value != _EVALUATE_STEER_SHED_MARKER
+ ]
+ if not candidates:
+ return None
+ candidates.sort(key=lambda item: item[1], reverse=True)
+ return candidates[0][0]
+
+
+def _fit_evaluate_steer_under_cap(result: dict[str, Any], data: dict[str, Any], *, is_repeat: bool) -> None:
+ """Keep the serialized result under the recent-output cap without ever head-slicing it.
+
+ A first evaluate is reconnaissance: if naming targets alone pushes it over cap, drop only the
+ advisory `actionable_targets` and leave the raw page (`data["result"]`) for the model to read.
+ A repeat is an imperative steer: the model already saw the page on evaluate #1, so shed the bulky
+ non-essential payload (the nested raw fields of `result`, then `result`/any large key wholesale,
+ replaced by a short marker) while always keeping `next_action`, its reason, and >=1 target."""
+
+ def over_cap() -> bool:
+ return len(json.dumps(result, default=str)) > _RECENT_TOOL_OUTPUT_CHAR_CAP
+
+ if not over_cap():
+ return
+ if not is_repeat:
+ data.pop("actionable_targets", None)
+ return
+ nested = data.get("result")
+ if isinstance(nested, dict):
+ for key in _EVALUATE_STEER_NESTED_BULKY_KEYS:
+ if key in nested and nested[key] != _EVALUATE_STEER_SHED_MARKER:
+ nested[key] = _EVALUATE_STEER_SHED_MARKER
+ if not over_cap():
+ return
+ while over_cap():
+ largest_key = _largest_non_essential_data_key(data)
+ if largest_key is None:
+ break
+ data[largest_key] = _EVALUATE_STEER_SHED_MARKER
+ if not over_cap():
+ return
+ targets = data.get("actionable_targets")
+ while isinstance(targets, list) and len(targets) > 1 and over_cap():
+ targets.pop()
+
+
+def _auto_act_essential_keys() -> frozenset[str]:
+ return _EVALUATE_STEER_ESSENTIAL_KEYS | {"auto_acted", "page"}
+
+
+async def _auto_act_on_repeat(ctx: AgentContext, result: dict[str, Any], *, url: str, target: dict[str, Any]) -> bool:
+ """Issue the in-process click for the single unambiguous target and reshape the result.
+
+ Returns True when the click landed and the result was reshaped to report it; False
+ degrades the caller to the advisory steer (next_action/actionable_targets left intact)."""
+ data = result.get("data")
+ if not isinstance(data, dict):
+ return False
+ server = getattr(ctx, "discovery_mcp_server", None)
+ if server is None:
+ return False
+ selector = target["selector"]
+ pre_url = await _live_working_page_url(ctx) or url
+ try:
+ click_result = await asyncio.wait_for(
+ server.call_internal_tool("skyvern_click", {"selector": selector, "selector_mode": "direct"}),
+ timeout=settings.COPILOT_SCOUT_ACT_OBSERVE_TIMEOUT_SECONDS,
+ )
+ except Exception:
+ LOG.warning("copilot_evaluate_auto_act_failed", url=pre_url, selector=selector, exc_info=True)
+ return False
+ if not isinstance(click_result, dict) or not click_result.get("ok"):
+ LOG.warning(
+ "copilot_evaluate_auto_act_failed",
+ url=pre_url,
+ selector=selector,
+ error=(click_result or {}).get("error") if isinstance(click_result, dict) else None,
+ )
+ return False
+
+ post_url = await _live_working_page_url(ctx) or url
+ post_evidence = await _scout_act_observe_page_evidence(ctx, url=post_url)
+ _record_scouted_interaction(ctx, tool_name="click", selector=selector, source_url=pre_url)
+
+ for key in ("next_action", "next_action_reason", "actionable_targets"):
+ data.pop(key, None)
+ data["auto_acted"] = {"tool": "click", "selector": selector, "text": target.get("text", "")}
+ if post_evidence is None:
+ data["auto_acted"]["note"] = "clicked; post-click page evidence unavailable"
+ else:
+ data["page"] = _build_scout_page_summary(post_evidence)
+ essential = _auto_act_essential_keys()
+ while len(json.dumps(result, default=str)) > _RECENT_TOOL_OUTPUT_CHAR_CAP:
+ largest = max(
+ (
+ (key, _serialized_len(value))
+ for key, value in data.items()
+ if key not in essential and value != _EVALUATE_STEER_SHED_MARKER
+ ),
+ key=lambda item: item[1],
+ default=None,
+ )
+ if largest is None:
+ page = data.get("page")
+ if not isinstance(page, dict) or not _shed_scout_page_summary_section(page):
+ break
+ continue
+ data[largest[0]] = _EVALUATE_STEER_SHED_MARKER
+ LOG.info("copilot_evaluate_auto_acted", url=post_url, selector=selector)
+ return True
+
+
+async def _maybe_steer_evaluate_to_action(ctx: AgentContext, result: dict[str, Any], *, url: str) -> None:
+ if not settings.COPILOT_EVALUATE_ACTIONABLE_TARGET_STEER_ENABLED:
+ return
+ data = result.get("data")
+ if not isinstance(data, dict):
+ return
+ try:
+ parsed = await _scout_act_observe_page_evidence(ctx, url=url)
+ if parsed is None or not has_bounded_page_schema(parsed):
+ _reset_evaluate_actionable_tracker(ctx)
+ return
+ identities = _actionable_target_identities(parsed)
+ if not identities:
+ _reset_evaluate_actionable_tracker(ctx)
+ return
+ signature = _actionable_target_signature(identities)
+ # Strict full-URL match (fragment included): on an SPA a hash-route change
+ # is a navigation, so a differing fragment must read as a different page.
+ is_repeat = ctx.last_evaluate_actionable_signature == signature and ctx.last_evaluate_actionable_url == url
+ ctx.last_evaluate_actionable_signature = signature
+ ctx.last_evaluate_actionable_url = url
+ if signature != ctx.last_auto_acted_signature and ctx.last_auto_acted_signature is not None:
+ ctx.last_auto_acted_signature = None
+ targets = _actionable_targets_for_result(identities)
+ if (
+ settings.COPILOT_EVALUATE_AUTO_ACT_ON_REPEAT_ENABLED
+ and is_repeat
+ and ctx.last_auto_acted_signature != signature
+ ):
+ candidate = _auto_act_candidate(parsed)
+ if candidate is not None:
+ ctx.last_auto_acted_signature = signature
+ if await _auto_act_on_repeat(ctx, result, url=url, target=candidate):
+ LOG.info("copilot_evaluate_actionable_target_steer", url=url, is_repeat=True, steered=True)
+ return
+ if targets:
+ data["actionable_targets"] = targets
+ if is_repeat:
+ data["next_action"] = "click"
+ data["next_action_reason"] = _EVALUATE_ACTIONABLE_ACT_INSTRUCTION
+ _fit_evaluate_steer_under_cap(result, data, is_repeat=is_repeat)
+ LOG.info(
+ "copilot_evaluate_actionable_target_steer",
+ url=url,
+ actionable_target_count=len(identities),
+ is_repeat=is_repeat,
+ steered=is_repeat and bool(targets),
+ )
+ except Exception:
+ data.pop("actionable_targets", None)
+ data.pop("next_action", None)
+ data.pop("next_action_reason", None)
+ _reset_evaluate_actionable_tracker(ctx)
+ LOG.warning("copilot_evaluate_actionable_target_steer_failed", exc_info=True)
+
+
def _mark_post_run_page_observed(ctx: AgentContext, *, source_tool: str, url: str) -> None:
run_id = getattr(ctx, "last_run_blocks_workflow_run_id", None)
if not isinstance(run_id, str) or not run_id:
diff --git a/tests/unit/test_copilot_hooks.py b/tests/unit/test_copilot_hooks.py
index 29cf7d88c..86e6fbe7b 100644
--- a/tests/unit/test_copilot_hooks.py
+++ b/tests/unit/test_copilot_hooks.py
@@ -10,8 +10,11 @@ from typing import Any
from unittest.mock import AsyncMock, MagicMock, call
import pytest
+from structlog.testing import capture_logs
+from skyvern.forge.sdk.copilot import hooks as hooks_module
from skyvern.forge.sdk.copilot.blocker_signal import CopilotToolBlockerSignal
+from skyvern.forge.sdk.copilot.enforcement import CopilotGoalSatisfied
from skyvern.forge.sdk.copilot.hooks import CopilotRunHooks
from skyvern.forge.sdk.copilot.turn_halt import CopilotTurnHalt, turn_halt_from_blocker_signal
@@ -19,6 +22,10 @@ from skyvern.forge.sdk.copilot.turn_halt import CopilotTurnHalt, turn_halt_from_
@dataclass
class _FakeContext:
tool_activity: list[dict[str, Any]] = field(default_factory=list)
+ workflow_permanent_id: str = "wpid_example"
+ turn_id: str = "turn_example"
+ workflow_copilot_chat_id: str = "chat_example"
+ total_tokens_used: int | None = None
# `on_tool_end(context, agent, tool, result)` only reads `tool` and `result`;
@@ -64,6 +71,21 @@ async def test_on_tool_end_appends_generic_tool_entry() -> None:
assert "output_preview" not in entry # non-whitelisted tool
+async def test_on_tool_end_logs_copilot_turn_identifiers() -> None:
+ ctx = _FakeContext(total_tokens_used=123)
+ hooks = CopilotRunHooks(ctx)
+
+ output = _mcp_text_output({"ok": True, "data": {"url": "https://example.com"}})
+ with capture_logs() as logs:
+ await hooks.on_tool_end(_UNUSED, _UNUSED, _fake_tool("navigate_browser"), output)
+
+ completed = next(log for log in logs if log["event"] == "copilot tool completed")
+ assert completed["workflow_permanent_id"] == "wpid_example"
+ assert completed["turn_id"] == "turn_example"
+ assert completed["workflow_copilot_chat_id"] == "chat_example"
+
+
@pytest.mark.asyncio
async def test_on_tool_end_raises_turn_halt_after_activity_recording() -> None:
ctx = _FakeContext()
@@ -193,12 +215,38 @@ async def test_on_tool_end_swallows_unserializable_output() -> None:
raise RuntimeError("str boom")
payload = {"ok": True, "data": {"blocks": [{"label": "bad", "output": _Unserializable()}]}}
- await hooks.on_tool_end(_UNUSED, _UNUSED, _fake_tool("run_blocks_and_collect_debug"), payload)
+ with capture_logs() as logs:
+ await hooks.on_tool_end(_UNUSED, _UNUSED, _fake_tool("run_blocks_and_collect_debug"), payload)
# The recording path raised inside json.dumps before append. The guard
# swallowed it, so the invariant is "the run did not crash" -- and the
# activity entry was dropped. That is the acceptable trade for observability.
assert ctx.tool_activity == []
+ warning = next(
+ log for log in logs if log["event"] == "CopilotRunHooks.on_tool_end recording failed, skipping entry"
+ )
+ assert warning["workflow_permanent_id"] == "wpid_example"
+ assert warning["turn_id"] == "turn_example"
+ assert warning["workflow_copilot_chat_id"] == "chat_example"
+
+
+async def test_on_tool_end_goal_satisfied_log_includes_copilot_turn_identifiers(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ ctx = _FakeContext()
+ hooks = CopilotRunHooks(ctx)
+ monkeypatch.setattr(hooks_module, "_tool_completion_satisfies_turn", lambda *_args: True)
+
+ output = _mcp_text_output({"ok": True, "data": {"workflow_run_id": "wrid_example"}})
+ with capture_logs() as logs:
+ with pytest.raises(CopilotGoalSatisfied):
+ await hooks.on_tool_end(_UNUSED, _UNUSED, _fake_tool("update_and_run_blocks"), output)
+
+ satisfied = next(log for log in logs if log["event"] == "copilot tool satisfied goal; stopping agent loop")
+ assert satisfied["workflow_permanent_id"] == "wpid_example"
+ assert satisfied["turn_id"] == "turn_example"
+ assert satisfied["workflow_copilot_chat_id"] == "chat_example"
class TestCopilotToCallToolResult:
diff --git a/tests/unit/test_evaluate_actionable_target.py b/tests/unit/test_evaluate_actionable_target.py
new file mode 100644
index 000000000..256c78c3a
--- /dev/null
+++ b/tests/unit/test_evaluate_actionable_target.py
@@ -0,0 +1,797 @@
+from __future__ import annotations
+
+import json
+
+import pytest
+
+from skyvern.config import settings
+from skyvern.forge.sdk.copilot.runtime import AgentContext
+from skyvern.forge.sdk.copilot.tools import scouting
+
+
+def _packet() -> dict:
+ return {
+ "navigation_targets": [
+ {"selector": "#print", "text": "View Printable Statement"},
+ {"selector": "#download", "text": "Download"},
+ ],
+ "forms": [],
+ "result_containers": [],
+ }
+
+
+def _full_packet() -> dict:
+ return {
+ "forms": [
+ {
+ "fields": [{"selector": "#user", "text": "Username"}],
+ "submit_controls": [{"selector": "#login", "text": "Log In"}],
+ }
+ ],
+ "navigation_targets": [{"selector": "#print", "text": "View Printable Statement"}],
+ "result_containers": [{"selector": "#results", "text": "Results"}],
+ "modal_overlays": [{"dismiss_controls": [{"selector": "#close", "text": "Close"}]}],
+ }
+
+
+def _ctx() -> AgentContext:
+ return AgentContext.__new__(AgentContext)
+
+
+async def _seed(ctx, monkeypatch, packet, url):
+ async def fake(c, *, url):
+ return packet
+
+ monkeypatch.setattr(scouting, "_scout_act_observe_page_evidence", fake)
+ result = {"ok": True, "data": {"url": url}}
+ await scouting._maybe_steer_evaluate_to_action(ctx, result, url=url)
+ return result
+
+
+def test_signature_stable_across_equal_identity_sets() -> None:
+ a = scouting._actionable_target_identities(_packet())
+ b = scouting._actionable_target_identities(_packet())
+ assert scouting._actionable_target_signature(a) == scouting._actionable_target_signature(b)
+
+
+def test_signature_changes_when_controls_change() -> None:
+ base = scouting._actionable_target_identities(_packet())
+ mutated_packet = _packet()
+ mutated_packet["navigation_targets"][0]["selector"] = "#print-v2"
+ mutated = scouting._actionable_target_identities(mutated_packet)
+ assert scouting._actionable_target_signature(base) != scouting._actionable_target_signature(mutated)
+
+
+async def test_first_evaluate_names_targets_without_next_action(monkeypatch) -> None:
+ monkeypatch.setattr(settings, "COPILOT_EVALUATE_ACTIONABLE_TARGET_STEER_ENABLED", True)
+
+ async def fake_evidence(ctx, *, url):
+ return _packet()
+
+ monkeypatch.setattr(scouting, "_scout_act_observe_page_evidence", fake_evidence)
+ ctx = _ctx()
+ ctx.last_evaluate_actionable_signature = None
+ ctx.last_evaluate_actionable_url = None
+ result = {"ok": True, "data": {"url": "https://example.com/bill"}}
+ await scouting._maybe_steer_evaluate_to_action(ctx, result, url="https://example.com/bill")
+ assert "actionable_targets" in result["data"]
+ assert "next_action" not in result["data"]
+
+
+async def test_repeat_evaluate_same_page_steers_to_click(monkeypatch) -> None:
+ monkeypatch.setattr(settings, "COPILOT_EVALUATE_ACTIONABLE_TARGET_STEER_ENABLED", True)
+
+ async def fake_evidence(ctx, *, url):
+ return _packet()
+
+ monkeypatch.setattr(scouting, "_scout_act_observe_page_evidence", fake_evidence)
+ ctx = _ctx()
+ ctx.last_evaluate_actionable_signature = None
+ ctx.last_evaluate_actionable_url = None
+ url = "https://example.com/bill"
+ first = {"ok": True, "data": {"url": url}}
+ await scouting._maybe_steer_evaluate_to_action(ctx, first, url=url)
+ second = {"ok": True, "data": {"url": url}}
+ await scouting._maybe_steer_evaluate_to_action(ctx, second, url=url)
+ assert second["data"].get("next_action") == "click"
+ assert second["data"]["actionable_targets"]
+
+
+async def test_changed_signature_not_steered(monkeypatch) -> None:
+ monkeypatch.setattr(settings, "COPILOT_EVALUATE_ACTIONABLE_TARGET_STEER_ENABLED", True)
+ packets = [
+ _packet(),
+ {"navigation_targets": [{"selector": "#other", "text": "Next"}], "forms": [], "result_containers": []},
+ ]
+ calls = {"i": 0}
+
+ async def fake_evidence(ctx, *, url):
+ packet = packets[calls["i"]]
+ calls["i"] += 1
+ return packet
+
+ monkeypatch.setattr(scouting, "_scout_act_observe_page_evidence", fake_evidence)
+ ctx = _ctx()
+ ctx.last_evaluate_actionable_signature = None
+ ctx.last_evaluate_actionable_url = None
+ url = "https://example.com/bill"
+ first = {"ok": True, "data": {"url": url}}
+ await scouting._maybe_steer_evaluate_to_action(ctx, first, url=url)
+ second = {"ok": True, "data": {"url": url}}
+ await scouting._maybe_steer_evaluate_to_action(ctx, second, url=url)
+ assert "next_action" not in second["data"]
+
+
+async def test_flag_off_restores_raw_result(monkeypatch) -> None:
+ monkeypatch.setattr(settings, "COPILOT_EVALUATE_ACTIONABLE_TARGET_STEER_ENABLED", False)
+ ctx = _ctx()
+ ctx.last_evaluate_actionable_signature = None
+ ctx.last_evaluate_actionable_url = None
+ result = {"ok": True, "data": {"url": "https://example.com/bill"}}
+ await scouting._maybe_steer_evaluate_to_action(ctx, result, url="https://example.com/bill")
+ assert "actionable_targets" not in result["data"]
+ assert "next_action" not in result["data"]
+
+
+async def test_same_signature_different_url_not_steered(monkeypatch) -> None:
+ monkeypatch.setattr(settings, "COPILOT_EVALUATE_ACTIONABLE_TARGET_STEER_ENABLED", True)
+ ctx = _ctx()
+ ctx.last_evaluate_actionable_signature = None
+ ctx.last_evaluate_actionable_url = None
+ await _seed(ctx, monkeypatch, _packet(), "https://app.example.com/a")
+ second = await _seed(ctx, monkeypatch, _packet(), "https://app.example.com/b")
+ assert "next_action" not in second["data"]
+
+
+async def test_hash_route_change_not_steered(monkeypatch) -> None:
+ monkeypatch.setattr(settings, "COPILOT_EVALUATE_ACTIONABLE_TARGET_STEER_ENABLED", True)
+ ctx = _ctx()
+ ctx.last_evaluate_actionable_signature = None
+ ctx.last_evaluate_actionable_url = None
+ await _seed(ctx, monkeypatch, _packet(), "https://app.example.com/wizard#/step1")
+ second = await _seed(ctx, monkeypatch, _packet(), "https://app.example.com/wizard#/step2")
+ assert "next_action" not in second["data"]
+
+
+async def test_repeat_scalar_result_sheds_under_cap(monkeypatch) -> None:
+ monkeypatch.setattr(settings, "COPILOT_EVALUATE_ACTIONABLE_TARGET_STEER_ENABLED", True)
+ ctx = _ctx()
+ ctx.last_evaluate_actionable_signature = None
+ ctx.last_evaluate_actionable_url = None
+ url = "https://app.example.com/bill"
+
+ async def fake(c, *, url):
+ return _packet()
+
+ monkeypatch.setattr(scouting, "_scout_act_observe_page_evidence", fake)
+ big = "x" * 6000
+ first = {"ok": True, "data": {"url": url, "result": big}}
+ await scouting._maybe_steer_evaluate_to_action(ctx, first, url=url)
+ second = {"ok": True, "data": {"url": url, "result": big}}
+ await scouting._maybe_steer_evaluate_to_action(ctx, second, url=url)
+ serialized = json.dumps(second, default=str)
+ assert second["data"].get("next_action") == "click"
+ assert second["data"].get("actionable_targets")
+ assert len(serialized) <= scouting._RECENT_TOOL_OUTPUT_CHAR_CAP
+ assert '"next_action"' in serialized[: scouting._RECENT_TOOL_OUTPUT_CHAR_CAP]
+
+
+async def test_repeat_nested_result_sheds_under_cap(monkeypatch) -> None:
+ monkeypatch.setattr(settings, "COPILOT_EVALUATE_ACTIONABLE_TARGET_STEER_ENABLED", True)
+ ctx = _ctx()
+ ctx.last_evaluate_actionable_signature = None
+ ctx.last_evaluate_actionable_url = None
+ url = "https://app.example.com/bill"
+
+ async def fake(c, *, url):
+ return _packet()
+
+ monkeypatch.setattr(scouting, "_scout_act_observe_page_evidence", fake)
+
+ def payload() -> dict:
+ return {
+ "url": url,
+ "result": {
+ "html": "<html>" + ("<div>row</div>" * 600) + "</html>",
+ "text": "May 5 2026 $4,210.55 " * 80,
+ "buttons": [f"button-{i}" for i in range(50)],
+ },
+ }
+
+ first = {"ok": True, "data": payload()}
+ assert len(json.dumps(first, default=str)) > 8000
+ await scouting._maybe_steer_evaluate_to_action(ctx, first, url=url)
+ second = {"ok": True, "data": payload()}
+ await scouting._maybe_steer_evaluate_to_action(ctx, second, url=url)
+ serialized = json.dumps(second, default=str)
+ assert second["data"].get("next_action") == "click"
+ assert second["data"].get("actionable_targets")
+ assert len(serialized) <= scouting._RECENT_TOOL_OUTPUT_CHAR_CAP
+ assert '"next_action"' in serialized[: scouting._RECENT_TOOL_OUTPUT_CHAR_CAP]
+
+
+async def test_first_evaluate_large_result_not_shed(monkeypatch) -> None:
+ monkeypatch.setattr(settings, "COPILOT_EVALUATE_ACTIONABLE_TARGET_STEER_ENABLED", True)
+ ctx = _ctx()
+ ctx.last_evaluate_actionable_signature = None
+ ctx.last_evaluate_actionable_url = None
+ url = "https://app.example.com/bill"
+
+ async def fake(c, *, url):
+ return _packet()
+
+ monkeypatch.setattr(scouting, "_scout_act_observe_page_evidence", fake)
+ big = "x" * 6000
+ first = {"ok": True, "data": {"url": url, "result": big}}
+ await scouting._maybe_steer_evaluate_to_action(ctx, first, url=url)
+ assert first["data"]["result"] == big
+ assert "next_action" not in first["data"]
+
+
+async def test_non_serializable_data_does_not_raise(monkeypatch) -> None:
+ monkeypatch.setattr(settings, "COPILOT_EVALUATE_ACTIONABLE_TARGET_STEER_ENABLED", True)
+ ctx = _ctx()
+ ctx.last_evaluate_actionable_signature = "seed"
+ ctx.last_evaluate_actionable_url = "https://app.example.com/bill"
+
+ async def fake(c, *, url):
+ raise TypeError("boom")
+
+ monkeypatch.setattr(scouting, "_scout_act_observe_page_evidence", fake)
+ result = {"ok": True, "data": {"url": "https://app.example.com/bill"}}
+ await scouting._maybe_steer_evaluate_to_action(ctx, result, url="https://app.example.com/bill")
+ assert "actionable_targets" not in result["data"]
+ assert "next_action" not in result["data"]
+ assert ctx.last_evaluate_actionable_signature is None
+
+
+def test_identities_cover_all_collections() -> None:
+ identities = scouting._actionable_target_identities(_full_packet())
+ selectors = {sel for sel, _ in identities}
+ assert {"#user", "#login", "#print", "#results", "#close"} <= selectors
+
+
+async def test_probe_none_leaves_data_untouched_and_resets(monkeypatch) -> None:
+ monkeypatch.setattr(settings, "COPILOT_EVALUATE_ACTIONABLE_TARGET_STEER_ENABLED", True)
+ ctx = _ctx()
+ ctx.last_evaluate_actionable_signature = "seed"
+ ctx.last_evaluate_actionable_url = "https://app.example.com/x"
+
+ async def fake(c, *, url):
+ return None
+
+ monkeypatch.setattr(scouting, "_scout_act_observe_page_evidence", fake)
+ result = {"ok": True, "data": {"url": "https://app.example.com/x"}}
+ await scouting._maybe_steer_evaluate_to_action(ctx, result, url="https://app.example.com/x")
+ assert result["data"] == {"url": "https://app.example.com/x"}
+ assert ctx.last_evaluate_actionable_signature is None
+
+
+async def test_probe_empty_collections_no_steer(monkeypatch) -> None:
+ monkeypatch.setattr(settings, "COPILOT_EVALUATE_ACTIONABLE_TARGET_STEER_ENABLED", True)
+ ctx = _ctx()
+ ctx.last_evaluate_actionable_signature = None
+ ctx.last_evaluate_actionable_url = None
+
+ async def fake(c, *, url):
+ return {"forms": [], "navigation_targets": [], "result_containers": []}
+
+ monkeypatch.setattr(scouting, "_scout_act_observe_page_evidence", fake)
+ result = {"ok": True, "data": {"url": "https://app.example.com/x"}}
+ await scouting._maybe_steer_evaluate_to_action(ctx, result, url="https://app.example.com/x")
+ assert "actionable_targets" not in result["data"]
+
+
+async def test_interaction_between_evaluates_suppresses_steer(monkeypatch) -> None:
+ monkeypatch.setattr(settings, "COPILOT_EVALUATE_ACTIONABLE_TARGET_STEER_ENABLED", True)
+ ctx = _ctx()
+ ctx.last_evaluate_actionable_signature = None
+ ctx.last_evaluate_actionable_url = None
+ ctx.scouted_interactions = []
+ ctx.scout_trajectory = []
+ url = "https://app.example.com/bill"
+ await _seed(ctx, monkeypatch, _packet(), url)
+ scouting._record_scouted_interaction(ctx, tool_name="click", selector="#print", source_url=url)
+ second = await _seed(ctx, monkeypatch, _packet(), url)
+ assert "next_action" not in second["data"]
+
+
+async def test_steer_front_runs_loop_guard(monkeypatch) -> None:
+ from skyvern.forge.sdk.copilot.loop_detection import MAX_CONSECUTIVE_SAME_TOOL, detect_tool_loop
+
+ monkeypatch.setattr(settings, "COPILOT_EVALUATE_ACTIONABLE_TARGET_STEER_ENABLED", True)
+ ctx = _ctx()
+ ctx.last_evaluate_actionable_signature = None
+ ctx.last_evaluate_actionable_url = None
+ url = "https://app.example.com/bill"
+ await _seed(ctx, monkeypatch, _packet(), url)
+ second = await _seed(ctx, monkeypatch, _packet(), url)
+ assert second["data"].get("next_action") == "click"
+
+ tracker: list[str] = []
+ assert detect_tool_loop(tracker, "evaluate") is None # evaluate #1 post-hook runs
+ assert detect_tool_loop(tracker, "evaluate") is None # evaluate #2 post-hook runs + steers
+ assert detect_tool_loop(tracker, "evaluate") is not None # evaluate #3 trips the guard
+ assert MAX_CONSECUTIVE_SAME_TOOL == 3
+
+
+class _FakeDiscoveryServer:
+ def __init__(self, click_result: dict) -> None:
+ self.click_result = click_result
+ self.calls: list[tuple[str, dict]] = []
+
+ async def call_internal_tool(self, tool_name: str, args: dict) -> dict:
+ self.calls.append((tool_name, args))
+ return self.click_result
+
+
+def _auto_act_ctx(server: _FakeDiscoveryServer) -> AgentContext:
+ ctx = _ctx()
+ ctx.last_evaluate_actionable_signature = None
+ ctx.last_evaluate_actionable_url = None
+ ctx.last_auto_acted_signature = None
+ ctx.browser_session_id = None
+ ctx.scouted_interactions = []
+ ctx.scout_trajectory = []
+ ctx.discovery_mcp_server = server
+ return ctx
+
+
+def _single_link_packet() -> dict:
+ return {
+ "forms": [],
+ "navigation_targets": [
+ {"selector": "#stmt", "text": "Download Statement", "href": "https://apexbiz.example.com/apexbiz.pdf"}
+ ],
+ "result_containers": [],
+ }
+
+
+def _two_link_packet() -> dict:
+ return {
+ "forms": [],
+ "navigation_targets": [
+ {"selector": "#a", "text": "First Doc", "href": "https://apexbiz.example.com/a.pdf"},
+ {"selector": "#b", "text": "Second Doc", "href": "https://apexbiz.example.com/b.pdf"},
+ ],
+ "result_containers": [],
+ }
+
+
+def _pay_submit_packet() -> dict:
+ return {
+ "forms": [
+ {
+ "method": "post",
+ "fields": [],
+ "submit_controls": [{"selector": "#pay", "text": "Pay $4,210.55", "type": "submit"}],
+ }
+ ],
+ "navigation_targets": [],
+ "result_containers": [],
+ }
+
+
+def _bare_button_submit_packet() -> dict:
+ # `<button>Continue</button>` (HTML-default submit) inside a method=post form:
+ # the structured-evidence producer reports type='button', so the writing-submit
+ # guard never fires. Nav-only candidacy is what keeps it from being auto-acted.
+ return {
+ "forms": [
+ {
+ "method": "post",
+ "fields": [],
+ "submit_controls": [{"selector": "#continue", "text": "Continue", "type": "button"}],
+ }
+ ],
+ "navigation_targets": [],
+ "result_containers": [],
+ }
+
+
+def _type_button_packet() -> dict:
+ return {
+ "forms": [
+ {
+ "method": "post",
+ "fields": [],
+ "submit_controls": [{"selector": "#go", "text": "Go", "type": "button"}],
+ }
+ ],
+ "navigation_targets": [],
+ "result_containers": [],
+ }
+
+
+def _empty_text_link_packet() -> dict:
+ return {
+ "forms": [],
+ "navigation_targets": [{"selector": "#icon", "text": " ", "href": "https://apexbiz.example.com/x"}],
+ "result_containers": [],
+ }
+
+
+def _post_click_packet() -> dict:
+ return {
+ "page_title": "Statement",
+ "forms": [],
+ "navigation_targets": [{"selector": "#back", "text": "Back", "href": "https://apexbiz.example.com/home"}],
+ "result_containers": [],
+ }
+
+
+def test_auto_act_candidate_rejects_multiple() -> None:
+ assert scouting._auto_act_candidate(_two_link_packet()) is None
+
+
+def test_auto_act_candidate_rejects_high_tier_submit() -> None:
+ assert scouting._auto_act_candidate(_pay_submit_packet()) is None
+
+
+def test_auto_act_candidate_rejects_bare_button_submit() -> None:
+ # A lone low-tier submit in a method=post form is a form submit, never a nav link.
+ assert scouting._auto_act_candidate(_bare_button_submit_packet()) is None
+
+
+def test_auto_act_candidate_rejects_type_button() -> None:
+ assert scouting._auto_act_candidate(_type_button_packet()) is None
+
+
+def test_auto_act_candidate_rejects_empty_text_link() -> None:
+ assert scouting._auto_act_candidate(_empty_text_link_packet()) is None
+
+
+def test_auto_act_candidate_accepts_single_low_tier_link() -> None:
+ candidate = scouting._auto_act_candidate(_single_link_packet())
+ assert candidate == {"selector": "#stmt", "text": "Download Statement"}
+
+
+def test_auto_act_candidate_rejects_javascript_href() -> None:
+ packet = _single_link_packet()
+ packet["navigation_targets"][0]["href"] = "javascript:void(0)"
+ assert scouting._auto_act_candidate(packet) is None
+
+
+async def test_multiple_candidates_no_auto_act(monkeypatch) -> None:
+ monkeypatch.setattr(settings, "COPILOT_EVALUATE_ACTIONABLE_TARGET_STEER_ENABLED", True)
+ monkeypatch.setattr(settings, "COPILOT_EVALUATE_AUTO_ACT_ON_REPEAT_ENABLED", True)
+ server = _FakeDiscoveryServer({"ok": True, "data": {}})
+ ctx = _auto_act_ctx(server)
+ url = "https://apexbiz.example.com/bill"
+
+ async def fake(c, *, url):
+ return _two_link_packet()
+
+ monkeypatch.setattr(scouting, "_scout_act_observe_page_evidence", fake)
+ first = {"ok": True, "data": {"url": url}}
+ await scouting._maybe_steer_evaluate_to_action(ctx, first, url=url)
+ second = {"ok": True, "data": {"url": url}}
+ await scouting._maybe_steer_evaluate_to_action(ctx, second, url=url)
+ assert server.calls == []
+ assert "auto_acted" not in second["data"]
+ assert second["data"].get("next_action") == "click"
+
+
+async def test_high_tier_no_auto_act(monkeypatch) -> None:
+ monkeypatch.setattr(settings, "COPILOT_EVALUATE_ACTIONABLE_TARGET_STEER_ENABLED", True)
+ monkeypatch.setattr(settings, "COPILOT_EVALUATE_AUTO_ACT_ON_REPEAT_ENABLED", True)
+ server = _FakeDiscoveryServer({"ok": True, "data": {}})
+ ctx = _auto_act_ctx(server)
+ url = "https://apexbiz.example.com/bill"
+
+ async def fake(c, *, url):
+ return _pay_submit_packet()
+
+ monkeypatch.setattr(scouting, "_scout_act_observe_page_evidence", fake)
+ first = {"ok": True, "data": {"url": url}}
+ await scouting._maybe_steer_evaluate_to_action(ctx, first, url=url)
+ second = {"ok": True, "data": {"url": url}}
+ await scouting._maybe_steer_evaluate_to_action(ctx, second, url=url)
+ assert server.calls == []
+ assert "auto_acted" not in second["data"]
+ assert second["data"].get("next_action") == "click"
+
+
+async def test_bare_button_submit_no_auto_act(monkeypatch) -> None:
+ monkeypatch.setattr(settings, "COPILOT_EVALUATE_ACTIONABLE_TARGET_STEER_ENABLED", True)
+ monkeypatch.setattr(settings, "COPILOT_EVALUATE_AUTO_ACT_ON_REPEAT_ENABLED", True)
+ server = _FakeDiscoveryServer({"ok": True, "data": {}})
+ ctx = _auto_act_ctx(server)
+ url = "https://apexbiz.example.com/checkout"
+
+ async def fake(c, *, url):
+ return _bare_button_submit_packet()
+
+ monkeypatch.setattr(scouting, "_scout_act_observe_page_evidence", fake)
+ first = {"ok": True, "data": {"url": url}}
+ await scouting._maybe_steer_evaluate_to_action(ctx, first, url=url)
+ second = {"ok": True, "data": {"url": url}}
+ await scouting._maybe_steer_evaluate_to_action(ctx, second, url=url)
+ assert server.calls == []
+ assert "auto_acted" not in second["data"]
+ assert second["data"].get("next_action") == "click"
+
+
+async def test_empty_text_link_no_auto_act(monkeypatch) -> None:
+ monkeypatch.setattr(settings, "COPILOT_EVALUATE_ACTIONABLE_TARGET_STEER_ENABLED", True)
+ monkeypatch.setattr(settings, "COPILOT_EVALUATE_AUTO_ACT_ON_REPEAT_ENABLED", True)
+ server = _FakeDiscoveryServer({"ok": True, "data": {}})
+ ctx = _auto_act_ctx(server)
+ url = "https://apexbiz.example.com/bill"
+
+ async def fake(c, *, url):
+ return _empty_text_link_packet()
+
+ monkeypatch.setattr(scouting, "_scout_act_observe_page_evidence", fake)
+ first = {"ok": True, "data": {"url": url}}
+ await scouting._maybe_steer_evaluate_to_action(ctx, first, url=url)
+ second = {"ok": True, "data": {"url": url}}
+ await scouting._maybe_steer_evaluate_to_action(ctx, second, url=url)
+ assert server.calls == []
+ assert "auto_acted" not in second["data"]
+ assert second["data"].get("next_action") == "click"
+
+
+async def test_single_low_tier_link_auto_acts(monkeypatch) -> None:
+ monkeypatch.setattr(settings, "COPILOT_EVALUATE_ACTIONABLE_TARGET_STEER_ENABLED", True)
+ monkeypatch.setattr(settings, "COPILOT_EVALUATE_AUTO_ACT_ON_REPEAT_ENABLED", True)
+ server = _FakeDiscoveryServer({"ok": True, "data": {"selector": "#stmt"}})
+ ctx = _auto_act_ctx(server)
+ url = "https://apexbiz.example.com/bill"
+ packets = [_single_link_packet(), _single_link_packet(), _post_click_packet()]
+ idx = {"i": 0}
+
+ async def fake(c, *, url):
+ packet = packets[idx["i"]]
+ idx["i"] += 1
+ return packet
+
+ monkeypatch.setattr(scouting, "_scout_act_observe_page_evidence", fake)
+ first = {"ok": True, "data": {"url": url}}
+ await scouting._maybe_steer_evaluate_to_action(ctx, first, url=url)
+ second = {"ok": True, "data": {"url": url}}
+ await scouting._maybe_steer_evaluate_to_action(ctx, second, url=url)
+ assert [call[0] for call in server.calls] == ["skyvern_click"]
+ assert server.calls[0][1] == {"selector": "#stmt", "selector_mode": "direct"}
+ assert second["data"]["auto_acted"] == {"tool": "click", "selector": "#stmt", "text": "Download Statement"}
+ assert second["data"]["page"]["page_title"] == "Statement"
+ assert "next_action" not in second["data"]
+ assert "actionable_targets" not in second["data"]
+ assert len(json.dumps(second, default=str)) <= scouting._RECENT_TOOL_OUTPUT_CHAR_CAP
+
+
+async def test_auto_act_post_evidence_none_sheds_bulky_result_under_cap(monkeypatch) -> None:
+ monkeypatch.setattr(settings, "COPILOT_EVALUATE_ACTIONABLE_TARGET_STEER_ENABLED", True)
+ monkeypatch.setattr(settings, "COPILOT_EVALUATE_AUTO_ACT_ON_REPEAT_ENABLED", True)
+ server = _FakeDiscoveryServer({"ok": True, "data": {"selector": "#stmt"}})
+ ctx = _auto_act_ctx(server)
+ url = "https://apexbiz.example.com/bill"
+ # First two calls feed the steer probe; the third is the post-click extractor (None).
+ evidence_queue: list[dict | None] = [_single_link_packet(), _single_link_packet(), None]
+ idx = {"i": 0}
+
+ async def fake(c, *, url):
+ packet = evidence_queue[idx["i"]]
+ idx["i"] += 1
+ return packet
+
+ monkeypatch.setattr(scouting, "_scout_act_observe_page_evidence", fake)
+ big = "x" * 10000
+ first = {"ok": True, "data": {"url": url, "result": big}}
+ await scouting._maybe_steer_evaluate_to_action(ctx, first, url=url)
+ second = {"ok": True, "data": {"url": url, "result": big}}
+ await scouting._maybe_steer_evaluate_to_action(ctx, second, url=url)
+ assert [call[0] for call in server.calls] == ["skyvern_click"]
+ assert "auto_acted" in second["data"]
+ assert second["data"]["auto_acted"]["selector"] == "#stmt"
+ assert second["data"]["auto_acted"].get("note")
+ assert "page" not in second["data"]
+ assert len(json.dumps(second, default=str)) <= scouting._RECENT_TOOL_OUTPUT_CHAR_CAP
+
+
+async def test_auto_act_success_branch_sheds_oversized_page_under_cap(monkeypatch) -> None:
+ monkeypatch.setattr(settings, "COPILOT_EVALUATE_ACTIONABLE_TARGET_STEER_ENABLED", True)
+ monkeypatch.setattr(settings, "COPILOT_EVALUATE_AUTO_ACT_ON_REPEAT_ENABLED", True)
+ server = _FakeDiscoveryServer({"ok": True, "data": {"selector": "#stmt"}})
+ ctx = _auto_act_ctx(server)
+ url = "https://apexbiz.example.com/bill"
+
+ def fat_post_click() -> dict:
+ return {
+ "page_title": "Statement",
+ "forms": [
+ {
+ "fields": [
+ {"selector": f"#f{i}", "label": f"Field number {i} with a long label"} for i in range(40)
+ ],
+ "submit_controls": [{"selector": f"#s{i}", "text": f"Submit option {i}"} for i in range(20)],
+ }
+ ],
+ "navigation_targets": [
+ {"selector": f"#n{i}", "text": f"Navigation link number {i} on the page"} for i in range(40)
+ ],
+ "result_containers": [],
+ "modal_overlays": [
+ {"dismiss_controls": [{"selector": f"#d{i}", "text": f"Dismiss control {i}"} for i in range(10)]}
+ ],
+ }
+
+ evidence_queue: list[dict] = [_single_link_packet(), _single_link_packet(), fat_post_click()]
+ idx = {"i": 0}
+
+ async def fake(c, *, url):
+ packet = evidence_queue[idx["i"]]
+ idx["i"] += 1
+ return packet
+
+ monkeypatch.setattr(scouting, "_scout_act_observe_page_evidence", fake)
+ big = "x" * 10000
+ first = {"ok": True, "data": {"url": url, "result": big}}
+ await scouting._maybe_steer_evaluate_to_action(ctx, first, url=url)
+ second = {"ok": True, "data": {"url": url, "result": big}}
+ await scouting._maybe_steer_evaluate_to_action(ctx, second, url=url)
+ assert [call[0] for call in server.calls] == ["skyvern_click"]
+ assert "auto_acted" in second["data"]
+ assert "page" in second["data"]
+ assert len(json.dumps(second, default=str)) <= scouting._RECENT_TOOL_OUTPUT_CHAR_CAP
+
+
+async def test_auto_act_re_arms_on_changed_signature(monkeypatch) -> None:
+ monkeypatch.setattr(settings, "COPILOT_EVALUATE_ACTIONABLE_TARGET_STEER_ENABLED", True)
+ monkeypatch.setattr(settings, "COPILOT_EVALUATE_AUTO_ACT_ON_REPEAT_ENABLED", True)
+ server = _FakeDiscoveryServer({"ok": True, "data": {"selector": "#stmt"}})
+ ctx = _auto_act_ctx(server)
+ url = "https://apexbiz.example.com/bill"
+
+ def second_page_link() -> dict:
+ return {
+ "forms": [],
+ "navigation_targets": [
+ {"selector": "#next-stmt", "text": "Next Statement", "href": "https://apexbiz.example.com/next.pdf"}
+ ],
+ "result_containers": [],
+ }
+
+ # eval1+eval2: page A (auto-acts) -> post-click evidence;
+ # eval3+eval4: page B (a genuinely changed page) re-arms and auto-acts again.
+ evidence_queue: list[dict] = [
+ _single_link_packet(),
+ _single_link_packet(),
+ _post_click_packet(),
+ second_page_link(),
+ second_page_link(),
+ _post_click_packet(),
+ ]
+ idx = {"i": 0}
+
+ async def fake(c, *, url):
+ packet = evidence_queue[idx["i"]]
+ idx["i"] += 1
+ return packet
+
+ monkeypatch.setattr(scouting, "_scout_act_observe_page_evidence", fake)
+ for _ in range(4):
+ result = {"ok": True, "data": {"url": url}}
+ await scouting._maybe_steer_evaluate_to_action(ctx, result, url=url)
+ clicked = [call[1]["selector"] for call in server.calls if call[0] == "skyvern_click"]
+ assert clicked == ["#stmt", "#next-stmt"]
+
+
+async def test_auto_act_idempotent_on_unchanged_signature(monkeypatch) -> None:
+ monkeypatch.setattr(settings, "COPILOT_EVALUATE_ACTIONABLE_TARGET_STEER_ENABLED", True)
+ monkeypatch.setattr(settings, "COPILOT_EVALUATE_AUTO_ACT_ON_REPEAT_ENABLED", True)
+ server = _FakeDiscoveryServer({"ok": True, "data": {"selector": "#stmt"}})
+ ctx = _auto_act_ctx(server)
+ url = "https://apexbiz.example.com/bill"
+
+ async def fake(c, *, url):
+ return _single_link_packet()
+
+ monkeypatch.setattr(scouting, "_scout_act_observe_page_evidence", fake)
+ for _ in range(3):
+ result = {"ok": True, "data": {"url": url}}
+ await scouting._maybe_steer_evaluate_to_action(ctx, result, url=url)
+ assert [call[0] for call in server.calls] == ["skyvern_click"]
+
+
+async def test_auto_act_flag_off_pure_advisory(monkeypatch) -> None:
+ monkeypatch.setattr(settings, "COPILOT_EVALUATE_ACTIONABLE_TARGET_STEER_ENABLED", True)
+ monkeypatch.setattr(settings, "COPILOT_EVALUATE_AUTO_ACT_ON_REPEAT_ENABLED", False)
+ server = _FakeDiscoveryServer({"ok": True, "data": {"selector": "#stmt"}})
+ ctx = _auto_act_ctx(server)
+ url = "https://apexbiz.example.com/bill"
+
+ async def fake(c, *, url):
+ return _single_link_packet()
+
+ monkeypatch.setattr(scouting, "_scout_act_observe_page_evidence", fake)
+ first = {"ok": True, "data": {"url": url}}
+ await scouting._maybe_steer_evaluate_to_action(ctx, first, url=url)
+ second = {"ok": True, "data": {"url": url}}
+ await scouting._maybe_steer_evaluate_to_action(ctx, second, url=url)
+ assert server.calls == []
+ assert "auto_acted" not in second["data"]
+ assert second["data"].get("next_action") == "click"
+
+
+async def test_auto_act_click_failure_degrades_to_advisory(monkeypatch) -> None:
+ monkeypatch.setattr(settings, "COPILOT_EVALUATE_ACTIONABLE_TARGET_STEER_ENABLED", True)
+ monkeypatch.setattr(settings, "COPILOT_EVALUATE_AUTO_ACT_ON_REPEAT_ENABLED", True)
+ server = _FakeDiscoveryServer({"ok": False, "error": "element not found"})
+ ctx = _auto_act_ctx(server)
+ url = "https://apexbiz.example.com/bill"
+
+ async def fake(c, *, url):
+ return _single_link_packet()
+
+ monkeypatch.setattr(scouting, "_scout_act_observe_page_evidence", fake)
+ first = {"ok": True, "data": {"url": url}}
+ await scouting._maybe_steer_evaluate_to_action(ctx, first, url=url)
+ second = {"ok": True, "data": {"url": url}}
+ await scouting._maybe_steer_evaluate_to_action(ctx, second, url=url)
+ assert [call[0] for call in server.calls] == ["skyvern_click"]
+ assert "auto_acted" not in second["data"]
+ assert second["data"].get("next_action") == "click"
+ assert second["data"].get("actionable_targets")
+
+
+async def test_auto_act_front_runs_the_third_evaluate(monkeypatch) -> None:
+ from skyvern.forge.sdk.copilot.loop_detection import detect_tool_loop
+
+ monkeypatch.setattr(settings, "COPILOT_EVALUATE_ACTIONABLE_TARGET_STEER_ENABLED", True)
+ monkeypatch.setattr(settings, "COPILOT_EVALUATE_AUTO_ACT_ON_REPEAT_ENABLED", True)
+ server = _FakeDiscoveryServer({"ok": True, "data": {"selector": "#stmt"}})
+ ctx = _auto_act_ctx(server)
+ url = "https://apexbiz.example.com/bill"
+ packets = [_single_link_packet(), _single_link_packet(), _post_click_packet()]
+ idx = {"i": 0}
+
+ async def fake(c, *, url):
+ packet = packets[idx["i"]]
+ idx["i"] += 1
+ return packet
+
+ monkeypatch.setattr(scouting, "_scout_act_observe_page_evidence", fake)
+ first = {"ok": True, "data": {"url": url}}
+ await scouting._maybe_steer_evaluate_to_action(ctx, first, url=url)
+ second = {"ok": True, "data": {"url": url}}
+ await scouting._maybe_steer_evaluate_to_action(ctx, second, url=url)
+ assert second["data"]["auto_acted"]["selector"] == "#stmt"
+ assert ctx.scouted_interactions and ctx.scouted_interactions[-1]["tool_name"] == "click"
+
+ # The auto-act click runs via call_internal_tool, which bypasses detect_tool_loop
+ # and never appends to the consecutive-tool tracker, so it does NOT reset the streak.
+ # The real front-run: the click changes the page, so evaluate #2 returns the post-click
+ # auto_acted result and the model sees new content rather than issuing a 3rd identical
+ # evaluate. If it did, SKY-10982's consecutive-same-tool guard still trips on evaluate #3
+ # as the safe backstop, shown here.
+ tripping: list[str] = []
+ detect_tool_loop(tripping, "evaluate")
+ detect_tool_loop(tripping, "evaluate")
+ assert detect_tool_loop(tripping, "evaluate") is not None