Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
SKY-10965 Add Copilot hook log attribution (#6578)
Andrew Neilsonae8fcddb2f4b
diff --git a/skyvern/config.py b/skyvern/config.pyindex e14212080..8218dddac 100644--- a/skyvern/config.py+++ b/skyvern/config.py@@ -157,6 +157,12 @@ class Settings(BaseSettings):# Off restores schema-less interaction packets plus standalone page inspects.COPILOT_SCOUT_ACT_OBSERVE_ENABLED: bool = TrueCOPILOT_SCOUT_ACT_OBSERVE_TIMEOUT_SECONDS: float = 4.0+ # Steer a repeat evaluate on an unchanged page toward acting on an already-identified target.+ # Off restores the raw evaluate result with no actionable-target naming or act instruction.+ COPILOT_EVALUATE_ACTIONABLE_TARGET_STEER_ENABLED: bool = True+ # On a repeat evaluate of an unchanged page with exactly one safe target, click it in-process+ # instead of only advising. Off (default) keeps the advisory-only steer above.+ COPILOT_EVALUATE_AUTO_ACT_ON_REPEAT_ENABLED: bool = False# Persist copilot completion criteria per goal epoch with tri-state verdicts# and definition-plane grading. Off restores per-turn re-derivation, the legacy# run-status success claim, and unknown-verdicts-as-unsatisfied consumer behavior.diff --git a/skyvern/forge/sdk/copilot/hooks.py b/skyvern/forge/sdk/copilot/hooks.pyindex 19441d913..6b88ce148 100644--- a/skyvern/forge/sdk/copilot/hooks.py+++ b/skyvern/forge/sdk/copilot/hooks.py@@ -48,6 +48,14 @@ _VERIFIED_GOAL_CONTEXT_ATTRS: frozenset[str] = frozenset()+def _copilot_log_fields(ctx: CopilotContext) -> dict[str, str | None]:+ return {+ "workflow_permanent_id": getattr(ctx, "workflow_permanent_id", None),+ "turn_id": getattr(ctx, "turn_id", None),+ "workflow_copilot_chat_id": getattr(ctx, "workflow_copilot_chat_id", None),+ }++def _tool_completion_satisfies_turn(ctx: CopilotContext, tool_name: str, parsed: Mapping[str, object]) -> bool:if tool_name not in {"run_blocks_and_collect_debug", "update_and_run_blocks"}:return False@@ -90,6 +98,7 @@ class CopilotRunHooks(RunHooksBase):ok=parsed.get("ok"),summary=summary,total_tokens=getattr(self._ctx, "total_tokens_used", None),+ **_copilot_log_fields(self._ctx),)if tool_name == "list_credentials" and parsed.get("ok"):@@ -124,6 +133,7 @@ class CopilotRunHooks(RunHooksBase):LOG.warning("CopilotRunHooks.on_tool_end recording failed, skipping entry",tool=getattr(tool, "name", None),+ **_copilot_log_fields(self._ctx),exc_info=True,)return@@ -142,6 +152,7 @@ class CopilotRunHooks(RunHooksBase):workflow_run_id=(parsed.get("data") or {}).get("workflow_run_id")if isinstance(parsed.get("data"), dict)else None,+ **_copilot_log_fields(self._ctx),)self._ctx.goal_satisfied_tool_name = tool_nameself._ctx.goal_satisfied_tool_output = dict(parsed)diff --git a/skyvern/forge/sdk/copilot/runtime.py b/skyvern/forge/sdk/copilot/runtime.pyindex 47ed80ced..2bff031d6 100644--- a/skyvern/forge/sdk/copilot/runtime.py+++ b/skyvern/forge/sdk/copilot/runtime.py@@ -247,6 +247,9 @@ class AgentContext:post_run_page_observation_workflow_run_id: str | None = Nonepost_run_page_observation_after_failed_test: bool = Falsepost_run_current_page_inspection_workflow_run_id: str | None = None+ last_evaluate_actionable_signature: str | None = None+ last_evaluate_actionable_url: str | None = None+ last_auto_acted_signature: str | None = Noneobserved_browser_urls: list[str] = field(default_factory=list)# Ephemeral within-turn scout captures; not persisted across turns.scouted_interactions: list[ScoutedInteraction] = field(default_factory=list)diff --git a/skyvern/forge/sdk/copilot/tools/mcp_hooks.py b/skyvern/forge/sdk/copilot/tools/mcp_hooks.pyindex 82bc2f931..3e3647236 100644--- a/skyvern/forge/sdk/copilot/tools/mcp_hooks.py+++ b/skyvern/forge/sdk/copilot/tools/mcp_hooks.py@@ -41,6 +41,7 @@ from .scouting import (_consume_scout_source_url,_mark_page_inspected,_mark_pending_browser_interaction_observation,+ _maybe_steer_evaluate_to_action,_record_scouted_interaction,_register_scout_interaction_observation,_resolve_scout_role_name,@@ -499,6 +500,7 @@ async def _evaluate_post_hook(title=title,observed_data=result["data"],)+ await _maybe_steer_evaluate_to_action(ctx, result, url=url)return resultdiff --git a/skyvern/forge/sdk/copilot/tools/scouting.py b/skyvern/forge/sdk/copilot/tools/scouting.pyindex a11c5fa91..ab17d9f25 100644--- a/skyvern/forge/sdk/copilot/tools/scouting.py+++ b/skyvern/forge/sdk/copilot/tools/scouting.py@@ -1,6 +1,7 @@from __future__ import annotationsimport asyncio+import hashlibimport jsonimport reimport time@@ -206,6 +207,7 @@ def _record_scouted_interaction(# press_key may be page-level, so it is recorded by key even with no selector; other tools require one.if tool_name != "press_key" and not selector:return+ _reset_evaluate_actionable_tracker(ctx)artifact: ScoutedInteraction = {"tool_name": tool_name}if selector:artifact["selector"] = selector@@ -449,6 +451,331 @@ def _attach_scout_page_summary(result: dict[str, Any], page_evidence: dict[str,LOG.warning("copilot_scout_act_observe_summary_failed", exc_info=True)+_EVALUATE_ACTIONABLE_MAX_TARGETS = 4++_EVALUATE_ACTIONABLE_ACT_INSTRUCTION = (+ "This page already exposes actionable targets; click the intended one rather than re-evaluating."+)+++def _reset_evaluate_actionable_tracker(ctx: AgentContext) -> None:+ ctx.last_evaluate_actionable_signature = None+ ctx.last_evaluate_actionable_url = None+++def _actionable_target_identities(evidence: dict[str, Any]) -> list[tuple[str, str]]:+ identities: list[tuple[str, str]] = []++ def add(control: Any) -> None:+ if not isinstance(control, dict):+ return+ selector = _summary_text(control.get("selector"))+ text = _summary_text(control.get("text") or control.get("value") or control.get("aria_label"))+ if selector or text:+ identities.append((selector, text))++ for form in evidence.get("forms") or []:+ if not isinstance(form, dict):+ continue+ for field_entry in form.get("fields") or []:+ add(field_entry)+ for control in form.get("submit_controls") or []:+ add(control)+ for target in evidence.get("navigation_targets") or []:+ add(target)+ for container in evidence.get("result_containers") or []:+ add(container)+ for overlay in evidence.get("modal_overlays") or []:+ if not isinstance(overlay, dict):+ continue+ for control in overlay.get("dismiss_controls") or []:+ add(control)+ return identities+++def _actionable_target_signature(identities: list[tuple[str, str]]) -> str:+ canonical = json.dumps(sorted(identities), separators=(",", ":"))+ return hashlib.sha256(canonical.encode("utf-8")).hexdigest()+++def _actionable_targets_for_result(identities: list[tuple[str, str]]) -> list[dict[str, str]]:+ targets: list[dict[str, str]] = []+ for selector, text in identities[:_EVALUATE_ACTIONABLE_MAX_TARGETS]:+ entry = {key: value for key, value in {"selector": selector, "text": text}.items() if value}+ if entry:+ targets.append(entry)+ return targets+++# Verbs that imply an irreversible or money-moving side effect — never auto-clicked.+_AUTO_ACT_HIGH_TIER_VERBS = frozenset(+ {+ "pay",+ "payment",+ "purchase",+ "buy",+ "order",+ "place order",+ "checkout",+ "delete",+ "remove",+ "transfer",+ "send",+ "submit payment",+ "confirm payment",+ "wire",+ "withdraw",+ "cancel",+ }+)+++def _auto_act_is_high_tier_label(*labels: Any) -> bool:+ for label in labels:+ if not isinstance(label, str):+ continue+ normalized = label.strip().lower()+ if not normalized:+ continue+ if any(verb in normalized for verb in _AUTO_ACT_HIGH_TIER_VERBS):+ return True+ return False+++def _auto_act_href_is_navigation(href: Any) -> bool:+ if not isinstance(href, str):+ return False+ candidate = href.strip()+ if not candidate or candidate.startswith("#"):+ return False+ lowered = candidate.lower()+ if lowered.startswith(("javascript:", "mailto:", "tel:")):+ return False+ if lowered.startswith(("http://", "https://")):+ return True+ return not lowered.startswith(("data:", "blob:"))+++def _auto_act_candidate(parsed: dict[str, Any]) -> dict[str, Any] | None:+ """Return the single unambiguous, low-tier nav link to auto-click, or None.++ Eligible only from navigation_targets (`<a href>` with a real http/https/relative+ href and non-empty text); form submit_controls, form fields, result containers, and+ modal dismiss controls are excluded. Form submits are never candidates because the+ structured-evidence producer cannot reliably distinguish a writing submit from a+ bare default-submit button, so the whole form-submit class is dropped for safety.+ Money-moving / destructive verbs are dropped. Exactly one survivor ⇒ act; zero or+ more than one ⇒ None."""+ candidates: list[dict[str, Any]] = []++ for target in parsed.get("navigation_targets") or []:+ if not isinstance(target, dict):+ continue+ selector = _summary_text(target.get("selector"))+ if not selector or target.get("disabled") is True:+ continue+ if not _auto_act_href_is_navigation(target.get("href")):+ continue+ text = _summary_text(target.get("text"))+ if not text:+ continue+ if _auto_act_is_high_tier_label(text, target.get("name"), target.get("id")):+ continue+ candidates.append({"selector": selector, "text": text})++ if len(candidates) != 1:+ return None+ return candidates[0]+++_EVALUATE_STEER_SHED_MARKER = "[omitted on repeat — act on the named target instead of re-reading]"++# Keys the steer must never shed: navigation/identity context plus the steer's own output.+_EVALUATE_STEER_ESSENTIAL_KEYS = frozenset(+ {"url", "title", "observation_step", "actionable_targets", "next_action", "next_action_reason"}+)++# Nested bulky subfields inside an evaluate `result` dict (the raw page payload).+_EVALUATE_STEER_NESTED_BULKY_KEYS = ("html", "outerHTML", "innerHTML", "body", "bodyText", "text", "buttons")+++def _serialized_len(value: Any) -> int:+ try:+ return len(json.dumps(value, default=str))+ except Exception:+ return len(str(value))+++def _largest_non_essential_data_key(data: dict[str, Any]) -> str | None:+ candidates = [+ (key, _serialized_len(value))+ for key, value in data.items()+ if key not in _EVALUATE_STEER_ESSENTIAL_KEYS and value != _EVALUATE_STEER_SHED_MARKER+ ]+ if not candidates:+ return None+ candidates.sort(key=lambda item: item[1], reverse=True)+ return candidates[0][0]+++def _fit_evaluate_steer_under_cap(result: dict[str, Any], data: dict[str, Any], *, is_repeat: bool) -> None:+ """Keep the serialized result under the recent-output cap without ever head-slicing it.++ A first evaluate is reconnaissance: if naming targets alone pushes it over cap, drop only the+ advisory `actionable_targets` and leave the raw page (`data["result"]`) for the model to read.+ A repeat is an imperative steer: the model already saw the page on evaluate #1, so shed the bulky+ non-essential payload (the nested raw fields of `result`, then `result`/any large key wholesale,+ replaced by a short marker) while always keeping `next_action`, its reason, and >=1 target."""++ def over_cap() -> bool:+ return len(json.dumps(result, default=str)) > _RECENT_TOOL_OUTPUT_CHAR_CAP++ if not over_cap():+ return+ if not is_repeat:+ data.pop("actionable_targets", None)+ return+ nested = data.get("result")+ if isinstance(nested, dict):+ for key in _EVALUATE_STEER_NESTED_BULKY_KEYS:+ if key in nested and nested[key] != _EVALUATE_STEER_SHED_MARKER:+ nested[key] = _EVALUATE_STEER_SHED_MARKER+ if not over_cap():+ return+ while over_cap():+ largest_key = _largest_non_essential_data_key(data)+ if largest_key is None:+ break+ data[largest_key] = _EVALUATE_STEER_SHED_MARKER+ if not over_cap():+ return+ targets = data.get("actionable_targets")+ while isinstance(targets, list) and len(targets) > 1 and over_cap():+ targets.pop()+++def _auto_act_essential_keys() -> frozenset[str]:+ return _EVALUATE_STEER_ESSENTIAL_KEYS | {"auto_acted", "page"}+++async def _auto_act_on_repeat(ctx: AgentContext, result: dict[str, Any], *, url: str, target: dict[str, Any]) -> bool:+ """Issue the in-process click for the single unambiguous target and reshape the result.++ Returns True when the click landed and the result was reshaped to report it; False+ degrades the caller to the advisory steer (next_action/actionable_targets left intact)."""+ data = result.get("data")+ if not isinstance(data, dict):+ return False+ server = getattr(ctx, "discovery_mcp_server", None)+ if server is None:+ return False+ selector = target["selector"]+ pre_url = await _live_working_page_url(ctx) or url+ try:+ click_result = await asyncio.wait_for(+ server.call_internal_tool("skyvern_click", {"selector": selector, "selector_mode": "direct"}),+ timeout=settings.COPILOT_SCOUT_ACT_OBSERVE_TIMEOUT_SECONDS,+ )+ except Exception:+ LOG.warning("copilot_evaluate_auto_act_failed", url=pre_url, selector=selector, exc_info=True)+ return False+ if not isinstance(click_result, dict) or not click_result.get("ok"):+ LOG.warning(+ "copilot_evaluate_auto_act_failed",+ url=pre_url,+ selector=selector,+ error=(click_result or {}).get("error") if isinstance(click_result, dict) else None,+ )+ return False++ post_url = await _live_working_page_url(ctx) or url+ post_evidence = await _scout_act_observe_page_evidence(ctx, url=post_url)+ _record_scouted_interaction(ctx, tool_name="click", selector=selector, source_url=pre_url)++ for key in ("next_action", "next_action_reason", "actionable_targets"):+ data.pop(key, None)+ data["auto_acted"] = {"tool": "click", "selector": selector, "text": target.get("text", "")}+ if post_evidence is None:+ data["auto_acted"]["note"] = "clicked; post-click page evidence unavailable"+ else:+ data["page"] = _build_scout_page_summary(post_evidence)+ essential = _auto_act_essential_keys()+ while len(json.dumps(result, default=str)) > _RECENT_TOOL_OUTPUT_CHAR_CAP:+ largest = max(+ (+ (key, _serialized_len(value))+ for key, value in data.items()+ if key not in essential and value != _EVALUATE_STEER_SHED_MARKER+ ),+ key=lambda item: item[1],+ default=None,+ )+ if largest is None:+ page = data.get("page")+ if not isinstance(page, dict) or not _shed_scout_page_summary_section(page):+ break+ continue+ data[largest[0]] = _EVALUATE_STEER_SHED_MARKER+ LOG.info("copilot_evaluate_auto_acted", url=post_url, selector=selector)+ return True+++async def _maybe_steer_evaluate_to_action(ctx: AgentContext, result: dict[str, Any], *, url: str) -> None:+ if not settings.COPILOT_EVALUATE_ACTIONABLE_TARGET_STEER_ENABLED:+ return+ data = result.get("data")+ if not isinstance(data, dict):+ return+ try:+ parsed = await _scout_act_observe_page_evidence(ctx, url=url)+ if parsed is None or not has_bounded_page_schema(parsed):+ _reset_evaluate_actionable_tracker(ctx)+ return+ identities = _actionable_target_identities(parsed)+ if not identities:+ _reset_evaluate_actionable_tracker(ctx)+ return+ signature = _actionable_target_signature(identities)+ # Strict full-URL match (fragment included): on an SPA a hash-route change+ # is a navigation, so a differing fragment must read as a different page.+ is_repeat = ctx.last_evaluate_actionable_signature == signature and ctx.last_evaluate_actionable_url == url+ ctx.last_evaluate_actionable_signature = signature+ ctx.last_evaluate_actionable_url = url+ if signature != ctx.last_auto_acted_signature and ctx.last_auto_acted_signature is not None:+ ctx.last_auto_acted_signature = None+ targets = _actionable_targets_for_result(identities)+ if (+ settings.COPILOT_EVALUATE_AUTO_ACT_ON_REPEAT_ENABLED+ and is_repeat+ and ctx.last_auto_acted_signature != signature+ ):+ candidate = _auto_act_candidate(parsed)+ if candidate is not None:+ ctx.last_auto_acted_signature = signature+ if await _auto_act_on_repeat(ctx, result, url=url, target=candidate):+ LOG.info("copilot_evaluate_actionable_target_steer", url=url, is_repeat=True, steered=True)+ return+ if targets:+ data["actionable_targets"] = targets+ if is_repeat:+ data["next_action"] = "click"+ data["next_action_reason"] = _EVALUATE_ACTIONABLE_ACT_INSTRUCTION+ _fit_evaluate_steer_under_cap(result, data, is_repeat=is_repeat)+ LOG.info(+ "copilot_evaluate_actionable_target_steer",+ url=url,+ actionable_target_count=len(identities),+ is_repeat=is_repeat,+ steered=is_repeat and bool(targets),+ )+ except Exception:+ data.pop("actionable_targets", None)+ data.pop("next_action", None)+ data.pop("next_action_reason", None)+ _reset_evaluate_actionable_tracker(ctx)+ LOG.warning("copilot_evaluate_actionable_target_steer_failed", exc_info=True)++def _mark_post_run_page_observed(ctx: AgentContext, *, source_tool: str, url: str) -> None:run_id = getattr(ctx, "last_run_blocks_workflow_run_id", None)if not isinstance(run_id, str) or not run_id:diff --git a/tests/unit/test_copilot_hooks.py b/tests/unit/test_copilot_hooks.pyindex 29cf7d88c..86e6fbe7b 100644--- a/tests/unit/test_copilot_hooks.py+++ b/tests/unit/test_copilot_hooks.py@@ -10,8 +10,11 @@ from typing import Anyfrom unittest.mock import AsyncMock, MagicMock, callimport pytest+from structlog.testing import capture_logs+from skyvern.forge.sdk.copilot import hooks as hooks_modulefrom skyvern.forge.sdk.copilot.blocker_signal import CopilotToolBlockerSignal+from skyvern.forge.sdk.copilot.enforcement import CopilotGoalSatisfiedfrom skyvern.forge.sdk.copilot.hooks import CopilotRunHooksfrom skyvern.forge.sdk.copilot.turn_halt import CopilotTurnHalt, turn_halt_from_blocker_signal@@ -19,6 +22,10 @@ from skyvern.forge.sdk.copilot.turn_halt import CopilotTurnHalt, turn_halt_from_@dataclassclass _FakeContext:tool_activity: list[dict[str, Any]] = field(default_factory=list)+ workflow_permanent_id: str = "wpid_example"+ turn_id: str = "turn_example"+ workflow_copilot_chat_id: str = "chat_example"+ total_tokens_used: int | None = None# `on_tool_end(context, agent, tool, result)` only reads `tool` and `result`;@@ -64,6 +71,21 @@ async def test_on_tool_end_appends_generic_tool_entry() -> None:assert "output_preview" not in entry # non-whitelisted tool+async def test_on_tool_end_logs_copilot_turn_identifiers() -> None:+ ctx = _FakeContext(total_tokens_used=123)+ hooks = CopilotRunHooks(ctx)++ output = _mcp_text_output({"ok": True, "data": {"url": "https://example.com"}})+ with capture_logs() as logs:+ await hooks.on_tool_end(_UNUSED, _UNUSED, _fake_tool("navigate_browser"), output)++ completed = next(log for log in logs if log["event"] == "copilot tool completed")+ assert completed["workflow_permanent_id"] == "wpid_example"+ assert completed["turn_id"] == "turn_example"+ assert completed["workflow_copilot_chat_id"] == "chat_example"++@pytest.mark.asyncioasync def test_on_tool_end_raises_turn_halt_after_activity_recording() -> None:ctx = _FakeContext()@@ -193,12 +215,38 @@ async def test_on_tool_end_swallows_unserializable_output() -> None:raise RuntimeError("str boom")payload = {"ok": True, "data": {"blocks": [{"label": "bad", "output": _Unserializable()}]}}- await hooks.on_tool_end(_UNUSED, _UNUSED, _fake_tool("run_blocks_and_collect_debug"), payload)+ with capture_logs() as logs:+ await hooks.on_tool_end(_UNUSED, _UNUSED, _fake_tool("run_blocks_and_collect_debug"), payload)# The recording path raised inside json.dumps before append. The guard# swallowed it, so the invariant is "the run did not crash" -- and the# activity entry was dropped. That is the acceptable trade for observability.assert ctx.tool_activity == []+ warning = next(+ log for log in logs if log["event"] == "CopilotRunHooks.on_tool_end recording failed, skipping entry"+ )+ assert warning["workflow_permanent_id"] == "wpid_example"+ assert warning["turn_id"] == "turn_example"+ assert warning["workflow_copilot_chat_id"] == "chat_example"+++async def test_on_tool_end_goal_satisfied_log_includes_copilot_turn_identifiers(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ ctx = _FakeContext()+ hooks = CopilotRunHooks(ctx)+ monkeypatch.setattr(hooks_module, "_tool_completion_satisfies_turn", lambda *_args: True)++ output = _mcp_text_output({"ok": True, "data": {"workflow_run_id": "wrid_example"}})+ with capture_logs() as logs:+ with pytest.raises(CopilotGoalSatisfied):+ await hooks.on_tool_end(_UNUSED, _UNUSED, _fake_tool("update_and_run_blocks"), output)++ satisfied = next(log for log in logs if log["event"] == "copilot tool satisfied goal; stopping agent loop")+ assert satisfied["workflow_permanent_id"] == "wpid_example"+ assert satisfied["turn_id"] == "turn_example"+ assert satisfied["workflow_copilot_chat_id"] == "chat_example"class TestCopilotToCallToolResult:diff --git a/tests/unit/test_evaluate_actionable_target.py b/tests/unit/test_evaluate_actionable_target.pynew file mode 100644index 000000000..256c78c3a--- /dev/null+++ b/tests/unit/test_evaluate_actionable_target.py@@ -0,0 +1,797 @@+from __future__ import annotations++import json++import pytest++from skyvern.config import settings+from skyvern.forge.sdk.copilot.runtime import AgentContext+from skyvern.forge.sdk.copilot.tools import scouting+++def _packet() -> dict:+ return {+ "navigation_targets": [+ {"selector": "#print", "text": "View Printable Statement"},+ {"selector": "#download", "text": "Download"},+ ],+ "forms": [],+ "result_containers": [],+ }+++def _full_packet() -> dict:+ return {+ "forms": [+ {+ "fields": [{"selector": "#user", "text": "Username"}],+ "submit_controls": [{"selector": "#login", "text": "Log In"}],+ }+ ],+ "navigation_targets": [{"selector": "#print", "text": "View Printable Statement"}],+ "result_containers": [{"selector": "#results", "text": "Results"}],+ "modal_overlays": [{"dismiss_controls": [{"selector": "#close", "text": "Close"}]}],+ }+++def _ctx() -> AgentContext:+ return AgentContext.__new__(AgentContext)+++async def _seed(ctx, monkeypatch, packet, url):+ async def fake(c, *, url):+ return packet++ monkeypatch.setattr(scouting, "_scout_act_observe_page_evidence", fake)+ result = {"ok": True, "data": {"url": url}}+ await scouting._maybe_steer_evaluate_to_action(ctx, result, url=url)+ return result+++def test_signature_stable_across_equal_identity_sets() -> None:+ a = scouting._actionable_target_identities(_packet())+ b = scouting._actionable_target_identities(_packet())+ assert scouting._actionable_target_signature(a) == scouting._actionable_target_signature(b)+++def test_signature_changes_when_controls_change() -> None:+ base = scouting._actionable_target_identities(_packet())+ mutated_packet = _packet()+ mutated_packet["navigation_targets"][0]["selector"] = "#print-v2"+ mutated = scouting._actionable_target_identities(mutated_packet)+ assert scouting._actionable_target_signature(base) != scouting._actionable_target_signature(mutated)+++async def test_first_evaluate_names_targets_without_next_action(monkeypatch) -> None:+ monkeypatch.setattr(settings, "COPILOT_EVALUATE_ACTIONABLE_TARGET_STEER_ENABLED", True)++ async def fake_evidence(ctx, *, url):+ return _packet()++ monkeypatch.setattr(scouting, "_scout_act_observe_page_evidence", fake_evidence)+ ctx = _ctx()+ ctx.last_evaluate_actionable_signature = None+ ctx.last_evaluate_actionable_url = None+ result = {"ok": True, "data": {"url": "https://example.com/bill"}}+ await scouting._maybe_steer_evaluate_to_action(ctx, result, url="https://example.com/bill")+ assert "actionable_targets" in result["data"]+ assert "next_action" not in result["data"]+++async def test_repeat_evaluate_same_page_steers_to_click(monkeypatch) -> None:+ monkeypatch.setattr(settings, "COPILOT_EVALUATE_ACTIONABLE_TARGET_STEER_ENABLED", True)++ async def fake_evidence(ctx, *, url):+ return _packet()++ monkeypatch.setattr(scouting, "_scout_act_observe_page_evidence", fake_evidence)+ ctx = _ctx()+ ctx.last_evaluate_actionable_signature = None+ ctx.last_evaluate_actionable_url = None+ url = "https://example.com/bill"+ first = {"ok": True, "data": {"url": url}}+ await scouting._maybe_steer_evaluate_to_action(ctx, first, url=url)+ second = {"ok": True, "data": {"url": url}}+ await scouting._maybe_steer_evaluate_to_action(ctx, second, url=url)+ assert second["data"].get("next_action") == "click"+ assert second["data"]["actionable_targets"]+++async def test_changed_signature_not_steered(monkeypatch) -> None:+ monkeypatch.setattr(settings, "COPILOT_EVALUATE_ACTIONABLE_TARGET_STEER_ENABLED", True)+ packets = [+ _packet(),+ {"navigation_targets": [{"selector": "#other", "text": "Next"}], "forms": [], "result_containers": []},+ ]+ calls = {"i": 0}++ async def fake_evidence(ctx, *, url):+ packet = packets[calls["i"]]+ calls["i"] += 1+ return packet++ monkeypatch.setattr(scouting, "_scout_act_observe_page_evidence", fake_evidence)+ ctx = _ctx()+ ctx.last_evaluate_actionable_signature = None+ ctx.last_evaluate_actionable_url = None+ url = "https://example.com/bill"+ first = {"ok": True, "data": {"url": url}}+ await scouting._maybe_steer_evaluate_to_action(ctx, first, url=url)+ second = {"ok": True, "data": {"url": url}}+ await scouting._maybe_steer_evaluate_to_action(ctx, second, url=url)+ assert "next_action" not in second["data"]+++async def test_flag_off_restores_raw_result(monkeypatch) -> None:+ monkeypatch.setattr(settings, "COPILOT_EVALUATE_ACTIONABLE_TARGET_STEER_ENABLED", False)+ ctx = _ctx()+ ctx.last_evaluate_actionable_signature = None+ ctx.last_evaluate_actionable_url = None+ result = {"ok": True, "data": {"url": "https://example.com/bill"}}+ await scouting._maybe_steer_evaluate_to_action(ctx, result, url="https://example.com/bill")+ assert "actionable_targets" not in result["data"]+ assert "next_action" not in result["data"]+++async def test_same_signature_different_url_not_steered(monkeypatch) -> None:+ monkeypatch.setattr(settings, "COPILOT_EVALUATE_ACTIONABLE_TARGET_STEER_ENABLED", True)+ ctx = _ctx()+ ctx.last_evaluate_actionable_signature = None+ ctx.last_evaluate_actionable_url = None+ await _seed(ctx, monkeypatch, _packet(), "https://app.example.com/a")+ second = await _seed(ctx, monkeypatch, _packet(), "https://app.example.com/b")+ assert "next_action" not in second["data"]+++async def test_hash_route_change_not_steered(monkeypatch) -> None:+ monkeypatch.setattr(settings, "COPILOT_EVALUATE_ACTIONABLE_TARGET_STEER_ENABLED", True)+ ctx = _ctx()+ ctx.last_evaluate_actionable_signature = None+ ctx.last_evaluate_actionable_url = None+ await _seed(ctx, monkeypatch, _packet(), "https://app.example.com/wizard#/step1")+ second = await _seed(ctx, monkeypatch, _packet(), "https://app.example.com/wizard#/step2")+ assert "next_action" not in second["data"]+++async def test_repeat_scalar_result_sheds_under_cap(monkeypatch) -> None:+ monkeypatch.setattr(settings, "COPILOT_EVALUATE_ACTIONABLE_TARGET_STEER_ENABLED", True)+ ctx = _ctx()+ ctx.last_evaluate_actionable_signature = None+ ctx.last_evaluate_actionable_url = None+ url = "https://app.example.com/bill"++ async def fake(c, *, url):+ return _packet()++ monkeypatch.setattr(scouting, "_scout_act_observe_page_evidence", fake)+ big = "x" * 6000+ first = {"ok": True, "data": {"url": url, "result": big}}+ await scouting._maybe_steer_evaluate_to_action(ctx, first, url=url)+ second = {"ok": True, "data": {"url": url, "result": big}}+ await scouting._maybe_steer_evaluate_to_action(ctx, second, url=url)+ serialized = json.dumps(second, default=str)+ assert second["data"].get("next_action") == "click"+ assert second["data"].get("actionable_targets")+ assert len(serialized) <= scouting._RECENT_TOOL_OUTPUT_CHAR_CAP+ assert '"next_action"' in serialized[: scouting._RECENT_TOOL_OUTPUT_CHAR_CAP]+++async def test_repeat_nested_result_sheds_under_cap(monkeypatch) -> None:+ monkeypatch.setattr(settings, "COPILOT_EVALUATE_ACTIONABLE_TARGET_STEER_ENABLED", True)+ ctx = _ctx()+ ctx.last_evaluate_actionable_signature = None+ ctx.last_evaluate_actionable_url = None+ url = "https://app.example.com/bill"++ async def fake(c, *, url):+ return _packet()++ monkeypatch.setattr(scouting, "_scout_act_observe_page_evidence", fake)++ def payload() -> dict:+ return {+ "url": url,+ "result": {+ "html": "<html>" + ("<div>row</div>" * 600) + "</html>",+ "text": "May 5 2026 $4,210.55 " * 80,+ "buttons": [f"button-{i}" for i in range(50)],+ },+ }++ first = {"ok": True, "data": payload()}+ assert len(json.dumps(first, default=str)) > 8000+ await scouting._maybe_steer_evaluate_to_action(ctx, first, url=url)+ second = {"ok": True, "data": payload()}+ await scouting._maybe_steer_evaluate_to_action(ctx, second, url=url)+ serialized = json.dumps(second, default=str)+ assert second["data"].get("next_action") == "click"+ assert second["data"].get("actionable_targets")+ assert len(serialized) <= scouting._RECENT_TOOL_OUTPUT_CHAR_CAP+ assert '"next_action"' in serialized[: scouting._RECENT_TOOL_OUTPUT_CHAR_CAP]+++async def test_first_evaluate_large_result_not_shed(monkeypatch) -> None:+ monkeypatch.setattr(settings, "COPILOT_EVALUATE_ACTIONABLE_TARGET_STEER_ENABLED", True)+ ctx = _ctx()+ ctx.last_evaluate_actionable_signature = None+ ctx.last_evaluate_actionable_url = None+ url = "https://app.example.com/bill"++ async def fake(c, *, url):+ return _packet()++ monkeypatch.setattr(scouting, "_scout_act_observe_page_evidence", fake)+ big = "x" * 6000+ first = {"ok": True, "data": {"url": url, "result": big}}+ await scouting._maybe_steer_evaluate_to_action(ctx, first, url=url)+ assert first["data"]["result"] == big+ assert "next_action" not in first["data"]+++async def test_non_serializable_data_does_not_raise(monkeypatch) -> None:+ monkeypatch.setattr(settings, "COPILOT_EVALUATE_ACTIONABLE_TARGET_STEER_ENABLED", True)+ ctx = _ctx()+ ctx.last_evaluate_actionable_signature = "seed"+ ctx.last_evaluate_actionable_url = "https://app.example.com/bill"++ async def fake(c, *, url):+ raise TypeError("boom")++ monkeypatch.setattr(scouting, "_scout_act_observe_page_evidence", fake)+ result = {"ok": True, "data": {"url": "https://app.example.com/bill"}}+ await scouting._maybe_steer_evaluate_to_action(ctx, result, url="https://app.example.com/bill")+ assert "actionable_targets" not in result["data"]+ assert "next_action" not in result["data"]+ assert ctx.last_evaluate_actionable_signature is None+++def test_identities_cover_all_collections() -> None:+ identities = scouting._actionable_target_identities(_full_packet())+ selectors = {sel for sel, _ in identities}+ assert {"#user", "#login", "#print", "#results", "#close"} <= selectors+++async def test_probe_none_leaves_data_untouched_and_resets(monkeypatch) -> None:+ monkeypatch.setattr(settings, "COPILOT_EVALUATE_ACTIONABLE_TARGET_STEER_ENABLED", True)+ ctx = _ctx()+ ctx.last_evaluate_actionable_signature = "seed"+ ctx.last_evaluate_actionable_url = "https://app.example.com/x"++ async def fake(c, *, url):+ return None++ monkeypatch.setattr(scouting, "_scout_act_observe_page_evidence", fake)+ result = {"ok": True, "data": {"url": "https://app.example.com/x"}}+ await scouting._maybe_steer_evaluate_to_action(ctx, result, url="https://app.example.com/x")+ assert result["data"] == {"url": "https://app.example.com/x"}+ assert ctx.last_evaluate_actionable_signature is None+++async def test_probe_empty_collections_no_steer(monkeypatch) -> None:+ monkeypatch.setattr(settings, "COPILOT_EVALUATE_ACTIONABLE_TARGET_STEER_ENABLED", True)+ ctx = _ctx()+ ctx.last_evaluate_actionable_signature = None+ ctx.last_evaluate_actionable_url = None++ async def fake(c, *, url):+ return {"forms": [], "navigation_targets": [], "result_containers": []}++ monkeypatch.setattr(scouting, "_scout_act_observe_page_evidence", fake)+ result = {"ok": True, "data": {"url": "https://app.example.com/x"}}+ await scouting._maybe_steer_evaluate_to_action(ctx, result, url="https://app.example.com/x")+ assert "actionable_targets" not in result["data"]+++async def test_interaction_between_evaluates_suppresses_steer(monkeypatch) -> None:+ monkeypatch.setattr(settings, "COPILOT_EVALUATE_ACTIONABLE_TARGET_STEER_ENABLED", True)+ ctx = _ctx()+ ctx.last_evaluate_actionable_signature = None+ ctx.last_evaluate_actionable_url = None+ ctx.scouted_interactions = []+ ctx.scout_trajectory = []+ url = "https://app.example.com/bill"+ await _seed(ctx, monkeypatch, _packet(), url)+ scouting._record_scouted_interaction(ctx, tool_name="click", selector="#print", source_url=url)+ second = await _seed(ctx, monkeypatch, _packet(), url)+ assert "next_action" not in second["data"]+++async def test_steer_front_runs_loop_guard(monkeypatch) -> None:+ from skyvern.forge.sdk.copilot.loop_detection import MAX_CONSECUTIVE_SAME_TOOL, detect_tool_loop++ monkeypatch.setattr(settings, "COPILOT_EVALUATE_ACTIONABLE_TARGET_STEER_ENABLED", True)+ ctx = _ctx()+ ctx.last_evaluate_actionable_signature = None+ ctx.last_evaluate_actionable_url = None+ url = "https://app.example.com/bill"+ await _seed(ctx, monkeypatch, _packet(), url)+ second = await _seed(ctx, monkeypatch, _packet(), url)+ assert second["data"].get("next_action") == "click"++ tracker: list[str] = []+ assert detect_tool_loop(tracker, "evaluate") is None # evaluate #1 post-hook runs+ assert detect_tool_loop(tracker, "evaluate") is None # evaluate #2 post-hook runs + steers+ assert detect_tool_loop(tracker, "evaluate") is not None # evaluate #3 trips the guard+ assert MAX_CONSECUTIVE_SAME_TOOL == 3+++class _FakeDiscoveryServer:+ def __init__(self, click_result: dict) -> None:+ self.click_result = click_result+ self.calls: list[tuple[str, dict]] = []++ async def call_internal_tool(self, tool_name: str, args: dict) -> dict:+ self.calls.append((tool_name, args))+ return self.click_result+++def _auto_act_ctx(server: _FakeDiscoveryServer) -> AgentContext:+ ctx = _ctx()+ ctx.last_evaluate_actionable_signature = None+ ctx.last_evaluate_actionable_url = None+ ctx.last_auto_acted_signature = None+ ctx.browser_session_id = None+ ctx.scouted_interactions = []+ ctx.scout_trajectory = []+ ctx.discovery_mcp_server = server+ return ctx+++def _single_link_packet() -> dict:+ return {+ "forms": [],+ "navigation_targets": [+ {"selector": "#stmt", "text": "Download Statement", "href": "https://apexbiz.example.com/apexbiz.pdf"}+ ],+ "result_containers": [],+ }+++def _two_link_packet() -> dict:+ return {+ "forms": [],+ "navigation_targets": [+ {"selector": "#a", "text": "First Doc", "href": "https://apexbiz.example.com/a.pdf"},+ {"selector": "#b", "text": "Second Doc", "href": "https://apexbiz.example.com/b.pdf"},+ ],+ "result_containers": [],+ }+++def _pay_submit_packet() -> dict:+ return {+ "forms": [+ {+ "method": "post",+ "fields": [],+ "submit_controls": [{"selector": "#pay", "text": "Pay $4,210.55", "type": "submit"}],+ }+ ],+ "navigation_targets": [],+ "result_containers": [],+ }+++def _bare_button_submit_packet() -> dict:+ # `<button>Continue</button>` (HTML-default submit) inside a method=post form:+ # the structured-evidence producer reports type='button', so the writing-submit+ # guard never fires. Nav-only candidacy is what keeps it from being auto-acted.+ return {+ "forms": [+ {+ "method": "post",+ "fields": [],+ "submit_controls": [{"selector": "#continue", "text": "Continue", "type": "button"}],+ }+ ],+ "navigation_targets": [],+ "result_containers": [],+ }+++def _type_button_packet() -> dict:+ return {+ "forms": [+ {+ "method": "post",+ "fields": [],+ "submit_controls": [{"selector": "#go", "text": "Go", "type": "button"}],+ }+ ],+ "navigation_targets": [],+ "result_containers": [],+ }+++def _empty_text_link_packet() -> dict:+ return {+ "forms": [],+ "navigation_targets": [{"selector": "#icon", "text": " ", "href": "https://apexbiz.example.com/x"}],+ "result_containers": [],+ }+++def _post_click_packet() -> dict:+ return {+ "page_title": "Statement",+ "forms": [],+ "navigation_targets": [{"selector": "#back", "text": "Back", "href": "https://apexbiz.example.com/home"}],+ "result_containers": [],+ }+++def test_auto_act_candidate_rejects_multiple() -> None:+ assert scouting._auto_act_candidate(_two_link_packet()) is None+++def test_auto_act_candidate_rejects_high_tier_submit() -> None:+ assert scouting._auto_act_candidate(_pay_submit_packet()) is None+++def test_auto_act_candidate_rejects_bare_button_submit() -> None:+ # A lone low-tier submit in a method=post form is a form submit, never a nav link.+ assert scouting._auto_act_candidate(_bare_button_submit_packet()) is None+++def test_auto_act_candidate_rejects_type_button() -> None:+ assert scouting._auto_act_candidate(_type_button_packet()) is None+++def test_auto_act_candidate_rejects_empty_text_link() -> None:+ assert scouting._auto_act_candidate(_empty_text_link_packet()) is None+++def test_auto_act_candidate_accepts_single_low_tier_link() -> None:+ candidate = scouting._auto_act_candidate(_single_link_packet())+ assert candidate == {"selector": "#stmt", "text": "Download Statement"}+++def test_auto_act_candidate_rejects_javascript_href() -> None:+ packet = _single_link_packet()+ packet["navigation_targets"][0]["href"] = "javascript:void(0)"+ assert scouting._auto_act_candidate(packet) is None+++async def test_multiple_candidates_no_auto_act(monkeypatch) -> None:+ monkeypatch.setattr(settings, "COPILOT_EVALUATE_ACTIONABLE_TARGET_STEER_ENABLED", True)+ monkeypatch.setattr(settings, "COPILOT_EVALUATE_AUTO_ACT_ON_REPEAT_ENABLED", True)+ server = _FakeDiscoveryServer({"ok": True, "data": {}})+ ctx = _auto_act_ctx(server)+ url = "https://apexbiz.example.com/bill"++ async def fake(c, *, url):+ return _two_link_packet()++ monkeypatch.setattr(scouting, "_scout_act_observe_page_evidence", fake)+ first = {"ok": True, "data": {"url": url}}+ await scouting._maybe_steer_evaluate_to_action(ctx, first, url=url)+ second = {"ok": True, "data": {"url": url}}+ await scouting._maybe_steer_evaluate_to_action(ctx, second, url=url)+ assert server.calls == []+ assert "auto_acted" not in second["data"]+ assert second["data"].get("next_action") == "click"+++async def test_high_tier_no_auto_act(monkeypatch) -> None:+ monkeypatch.setattr(settings, "COPILOT_EVALUATE_ACTIONABLE_TARGET_STEER_ENABLED", True)+ monkeypatch.setattr(settings, "COPILOT_EVALUATE_AUTO_ACT_ON_REPEAT_ENABLED", True)+ server = _FakeDiscoveryServer({"ok": True, "data": {}})+ ctx = _auto_act_ctx(server)+ url = "https://apexbiz.example.com/bill"++ async def fake(c, *, url):+ return _pay_submit_packet()++ monkeypatch.setattr(scouting, "_scout_act_observe_page_evidence", fake)+ first = {"ok": True, "data": {"url": url}}+ await scouting._maybe_steer_evaluate_to_action(ctx, first, url=url)+ second = {"ok": True, "data": {"url": url}}+ await scouting._maybe_steer_evaluate_to_action(ctx, second, url=url)+ assert server.calls == []+ assert "auto_acted" not in second["data"]+ assert second["data"].get("next_action") == "click"+++async def test_bare_button_submit_no_auto_act(monkeypatch) -> None:+ monkeypatch.setattr(settings, "COPILOT_EVALUATE_ACTIONABLE_TARGET_STEER_ENABLED", True)+ monkeypatch.setattr(settings, "COPILOT_EVALUATE_AUTO_ACT_ON_REPEAT_ENABLED", True)+ server = _FakeDiscoveryServer({"ok": True, "data": {}})+ ctx = _auto_act_ctx(server)+ url = "https://apexbiz.example.com/checkout"++ async def fake(c, *, url):+ return _bare_button_submit_packet()++ monkeypatch.setattr(scouting, "_scout_act_observe_page_evidence", fake)+ first = {"ok": True, "data": {"url": url}}+ await scouting._maybe_steer_evaluate_to_action(ctx, first, url=url)+ second = {"ok": True, "data": {"url": url}}+ await scouting._maybe_steer_evaluate_to_action(ctx, second, url=url)+ assert server.calls == []+ assert "auto_acted" not in second["data"]+ assert second["data"].get("next_action") == "click"+++async def test_empty_text_link_no_auto_act(monkeypatch) -> None:+ monkeypatch.setattr(settings, "COPILOT_EVALUATE_ACTIONABLE_TARGET_STEER_ENABLED", True)+ monkeypatch.setattr(settings, "COPILOT_EVALUATE_AUTO_ACT_ON_REPEAT_ENABLED", True)+ server = _FakeDiscoveryServer({"ok": True, "data": {}})+ ctx = _auto_act_ctx(server)+ url = "https://apexbiz.example.com/bill"++ async def fake(c, *, url):+ return _empty_text_link_packet()++ monkeypatch.setattr(scouting, "_scout_act_observe_page_evidence", fake)+ first = {"ok": True, "data": {"url": url}}+ await scouting._maybe_steer_evaluate_to_action(ctx, first, url=url)+ second = {"ok": True, "data": {"url": url}}+ await scouting._maybe_steer_evaluate_to_action(ctx, second, url=url)+ assert server.calls == []+ assert "auto_acted" not in second["data"]+ assert second["data"].get("next_action") == "click"+++async def test_single_low_tier_link_auto_acts(monkeypatch) -> None:+ monkeypatch.setattr(settings, "COPILOT_EVALUATE_ACTIONABLE_TARGET_STEER_ENABLED", True)+ monkeypatch.setattr(settings, "COPILOT_EVALUATE_AUTO_ACT_ON_REPEAT_ENABLED", True)+ server = _FakeDiscoveryServer({"ok": True, "data": {"selector": "#stmt"}})+ ctx = _auto_act_ctx(server)+ url = "https://apexbiz.example.com/bill"+ packets = [_single_link_packet(), _single_link_packet(), _post_click_packet()]+ idx = {"i": 0}++ async def fake(c, *, url):+ packet = packets[idx["i"]]+ idx["i"] += 1+ return packet++ monkeypatch.setattr(scouting, "_scout_act_observe_page_evidence", fake)+ first = {"ok": True, "data": {"url": url}}+ await scouting._maybe_steer_evaluate_to_action(ctx, first, url=url)+ second = {"ok": True, "data": {"url": url}}+ await scouting._maybe_steer_evaluate_to_action(ctx, second, url=url)+ assert [call[0] for call in server.calls] == ["skyvern_click"]+ assert server.calls[0][1] == {"selector": "#stmt", "selector_mode": "direct"}+ assert second["data"]["auto_acted"] == {"tool": "click", "selector": "#stmt", "text": "Download Statement"}+ assert second["data"]["page"]["page_title"] == "Statement"+ assert "next_action" not in second["data"]+ assert "actionable_targets" not in second["data"]+ assert len(json.dumps(second, default=str)) <= scouting._RECENT_TOOL_OUTPUT_CHAR_CAP+++async def test_auto_act_post_evidence_none_sheds_bulky_result_under_cap(monkeypatch) -> None:+ monkeypatch.setattr(settings, "COPILOT_EVALUATE_ACTIONABLE_TARGET_STEER_ENABLED", True)+ monkeypatch.setattr(settings, "COPILOT_EVALUATE_AUTO_ACT_ON_REPEAT_ENABLED", True)+ server = _FakeDiscoveryServer({"ok": True, "data": {"selector": "#stmt"}})+ ctx = _auto_act_ctx(server)+ url = "https://apexbiz.example.com/bill"+ # First two calls feed the steer probe; the third is the post-click extractor (None).+ evidence_queue: list[dict | None] = [_single_link_packet(), _single_link_packet(), None]+ idx = {"i": 0}++ async def fake(c, *, url):+ packet = evidence_queue[idx["i"]]+ idx["i"] += 1+ return packet++ monkeypatch.setattr(scouting, "_scout_act_observe_page_evidence", fake)+ big = "x" * 10000+ first = {"ok": True, "data": {"url": url, "result": big}}+ await scouting._maybe_steer_evaluate_to_action(ctx, first, url=url)+ second = {"ok": True, "data": {"url": url, "result": big}}+ await scouting._maybe_steer_evaluate_to_action(ctx, second, url=url)+ assert [call[0] for call in server.calls] == ["skyvern_click"]+ assert "auto_acted" in second["data"]+ assert second["data"]["auto_acted"]["selector"] == "#stmt"+ assert second["data"]["auto_acted"].get("note")+ assert "page" not in second["data"]+ assert len(json.dumps(second, default=str)) <= scouting._RECENT_TOOL_OUTPUT_CHAR_CAP+++async def test_auto_act_success_branch_sheds_oversized_page_under_cap(monkeypatch) -> None:+ monkeypatch.setattr(settings, "COPILOT_EVALUATE_ACTIONABLE_TARGET_STEER_ENABLED", True)+ monkeypatch.setattr(settings, "COPILOT_EVALUATE_AUTO_ACT_ON_REPEAT_ENABLED", True)+ server = _FakeDiscoveryServer({"ok": True, "data": {"selector": "#stmt"}})+ ctx = _auto_act_ctx(server)+ url = "https://apexbiz.example.com/bill"++ def fat_post_click() -> dict:+ return {+ "page_title": "Statement",+ "forms": [+ {+ "fields": [+ {"selector": f"#f{i}", "label": f"Field number {i} with a long label"} for i in range(40)+ ],+ "submit_controls": [{"selector": f"#s{i}", "text": f"Submit option {i}"} for i in range(20)],+ }+ ],+ "navigation_targets": [+ {"selector": f"#n{i}", "text": f"Navigation link number {i} on the page"} for i in range(40)+ ],+ "result_containers": [],+ "modal_overlays": [+ {"dismiss_controls": [{"selector": f"#d{i}", "text": f"Dismiss control {i}"} for i in range(10)]}+ ],+ }++ evidence_queue: list[dict] = [_single_link_packet(), _single_link_packet(), fat_post_click()]+ idx = {"i": 0}++ async def fake(c, *, url):+ packet = evidence_queue[idx["i"]]+ idx["i"] += 1+ return packet++ monkeypatch.setattr(scouting, "_scout_act_observe_page_evidence", fake)+ big = "x" * 10000+ first = {"ok": True, "data": {"url": url, "result": big}}+ await scouting._maybe_steer_evaluate_to_action(ctx, first, url=url)+ second = {"ok": True, "data": {"url": url, "result": big}}+ await scouting._maybe_steer_evaluate_to_action(ctx, second, url=url)+ assert [call[0] for call in server.calls] == ["skyvern_click"]+ assert "auto_acted" in second["data"]+ assert "page" in second["data"]+ assert len(json.dumps(second, default=str)) <= scouting._RECENT_TOOL_OUTPUT_CHAR_CAP+++async def test_auto_act_re_arms_on_changed_signature(monkeypatch) -> None:+ monkeypatch.setattr(settings, "COPILOT_EVALUATE_ACTIONABLE_TARGET_STEER_ENABLED", True)+ monkeypatch.setattr(settings, "COPILOT_EVALUATE_AUTO_ACT_ON_REPEAT_ENABLED", True)+ server = _FakeDiscoveryServer({"ok": True, "data": {"selector": "#stmt"}})+ ctx = _auto_act_ctx(server)+ url = "https://apexbiz.example.com/bill"++ def second_page_link() -> dict:+ return {+ "forms": [],+ "navigation_targets": [+ {"selector": "#next-stmt", "text": "Next Statement", "href": "https://apexbiz.example.com/next.pdf"}+ ],+ "result_containers": [],+ }++ # eval1+eval2: page A (auto-acts) -> post-click evidence;+ # eval3+eval4: page B (a genuinely changed page) re-arms and auto-acts again.+ evidence_queue: list[dict] = [+ _single_link_packet(),+ _single_link_packet(),+ _post_click_packet(),+ second_page_link(),+ second_page_link(),+ _post_click_packet(),+ ]+ idx = {"i": 0}++ async def fake(c, *, url):+ packet = evidence_queue[idx["i"]]+ idx["i"] += 1+ return packet++ monkeypatch.setattr(scouting, "_scout_act_observe_page_evidence", fake)+ for _ in range(4):+ result = {"ok": True, "data": {"url": url}}+ await scouting._maybe_steer_evaluate_to_action(ctx, result, url=url)+ clicked = [call[1]["selector"] for call in server.calls if call[0] == "skyvern_click"]+ assert clicked == ["#stmt", "#next-stmt"]+++async def test_auto_act_idempotent_on_unchanged_signature(monkeypatch) -> None:+ monkeypatch.setattr(settings, "COPILOT_EVALUATE_ACTIONABLE_TARGET_STEER_ENABLED", True)+ monkeypatch.setattr(settings, "COPILOT_EVALUATE_AUTO_ACT_ON_REPEAT_ENABLED", True)+ server = _FakeDiscoveryServer({"ok": True, "data": {"selector": "#stmt"}})+ ctx = _auto_act_ctx(server)+ url = "https://apexbiz.example.com/bill"++ async def fake(c, *, url):+ return _single_link_packet()++ monkeypatch.setattr(scouting, "_scout_act_observe_page_evidence", fake)+ for _ in range(3):+ result = {"ok": True, "data": {"url": url}}+ await scouting._maybe_steer_evaluate_to_action(ctx, result, url=url)+ assert [call[0] for call in server.calls] == ["skyvern_click"]+++async def test_auto_act_flag_off_pure_advisory(monkeypatch) -> None:+ monkeypatch.setattr(settings, "COPILOT_EVALUATE_ACTIONABLE_TARGET_STEER_ENABLED", True)+ monkeypatch.setattr(settings, "COPILOT_EVALUATE_AUTO_ACT_ON_REPEAT_ENABLED", False)+ server = _FakeDiscoveryServer({"ok": True, "data": {"selector": "#stmt"}})+ ctx = _auto_act_ctx(server)+ url = "https://apexbiz.example.com/bill"++ async def fake(c, *, url):+ return _single_link_packet()++ monkeypatch.setattr(scouting, "_scout_act_observe_page_evidence", fake)+ first = {"ok": True, "data": {"url": url}}+ await scouting._maybe_steer_evaluate_to_action(ctx, first, url=url)+ second = {"ok": True, "data": {"url": url}}+ await scouting._maybe_steer_evaluate_to_action(ctx, second, url=url)+ assert server.calls == []+ assert "auto_acted" not in second["data"]+ assert second["data"].get("next_action") == "click"+++async def test_auto_act_click_failure_degrades_to_advisory(monkeypatch) -> None:+ monkeypatch.setattr(settings, "COPILOT_EVALUATE_ACTIONABLE_TARGET_STEER_ENABLED", True)+ monkeypatch.setattr(settings, "COPILOT_EVALUATE_AUTO_ACT_ON_REPEAT_ENABLED", True)+ server = _FakeDiscoveryServer({"ok": False, "error": "element not found"})+ ctx = _auto_act_ctx(server)+ url = "https://apexbiz.example.com/bill"++ async def fake(c, *, url):+ return _single_link_packet()++ monkeypatch.setattr(scouting, "_scout_act_observe_page_evidence", fake)+ first = {"ok": True, "data": {"url": url}}+ await scouting._maybe_steer_evaluate_to_action(ctx, first, url=url)+ second = {"ok": True, "data": {"url": url}}+ await scouting._maybe_steer_evaluate_to_action(ctx, second, url=url)+ assert [call[0] for call in server.calls] == ["skyvern_click"]+ assert "auto_acted" not in second["data"]+ assert second["data"].get("next_action") == "click"+ assert second["data"].get("actionable_targets")+++async def test_auto_act_front_runs_the_third_evaluate(monkeypatch) -> None:+ from skyvern.forge.sdk.copilot.loop_detection import detect_tool_loop++ monkeypatch.setattr(settings, "COPILOT_EVALUATE_ACTIONABLE_TARGET_STEER_ENABLED", True)+ monkeypatch.setattr(settings, "COPILOT_EVALUATE_AUTO_ACT_ON_REPEAT_ENABLED", True)+ server = _FakeDiscoveryServer({"ok": True, "data": {"selector": "#stmt"}})+ ctx = _auto_act_ctx(server)+ url = "https://apexbiz.example.com/bill"+ packets = [_single_link_packet(), _single_link_packet(), _post_click_packet()]+ idx = {"i": 0}++ async def fake(c, *, url):+ packet = packets[idx["i"]]+ idx["i"] += 1+ return packet++ monkeypatch.setattr(scouting, "_scout_act_observe_page_evidence", fake)+ first = {"ok": True, "data": {"url": url}}+ await scouting._maybe_steer_evaluate_to_action(ctx, first, url=url)+ second = {"ok": True, "data": {"url": url}}+ await scouting._maybe_steer_evaluate_to_action(ctx, second, url=url)+ assert second["data"]["auto_acted"]["selector"] == "#stmt"+ assert ctx.scouted_interactions and ctx.scouted_interactions[-1]["tool_name"] == "click"++ # The auto-act click runs via call_internal_tool, which bypasses detect_tool_loop+ # and never appends to the consecutive-tool tracker, so it does NOT reset the streak.+ # The real front-run: the click changes the page, so evaluate #2 returns the post-click+ # auto_acted result and the model sees new content rather than issuing a 3rd identical+ # evaluate. If it did, SKY-10982's consecutive-same-tool guard still trips on evaluate #3+ # as the safe backstop, shown here.+ tripping: list[str] = []+ detect_tool_loop(tripping, "evaluate")+ detect_tool_loop(tripping, "evaluate")+ assert detect_tool_loop(tripping, "evaluate") is not None