Skip to content

Automate browser based workflows with AI

SKY-11182: steer loaded results into composition (#6723)

Andrew Neilson9d6c5f11f2ca
diff --git a/skyvern/forge/sdk/copilot/blocker_signal.py b/skyvern/forge/sdk/copilot/blocker_signal.py
index 1f9978c3a..d2a460844 100644
--- a/skyvern/forge/sdk/copilot/blocker_signal.py
+++ b/skyvern/forge/sdk/copilot/blocker_signal.py
@@ -12,6 +12,7 @@ import structlog
from pydantic import BaseModel, ConfigDict, Field, field_serializer, model_validator
from skyvern.forge.sdk.copilot.failure_tracking import ACTIVE_RUN_TERMINAL_EVIDENCE_REASON_CODE
+from skyvern.forge.sdk.copilot.result_evidence import LoadedResultCompositionEvidence
from skyvern.forge.sdk.copilot.run_outcome import RecordedRunOutcome
BlockerKind = Literal[
@@ -171,7 +172,7 @@ class TerminalEvidence:
@dataclass(frozen=True)
class LoopBlockerEvidence(TerminalEvidence):
- pass
+ latest_evaluate_result_composition_steer: LoadedResultCompositionEvidence | None = None
class _LoopEvidenceCtx(Protocol):
@@ -183,6 +184,7 @@ class _LoopEvidenceCtx(Protocol):
staged_workflow: Any | None
staged_workflow_yaml: str | None
has_staged_proposal: bool
+ latest_evaluate_result_composition_steer: LoadedResultCompositionEvidence | None
class _BlockerSignalCtx(_LoopEvidenceCtx, Protocol):
@@ -239,6 +241,8 @@ def clear_terminal_evidence_on_workflow_edit(ctx: _TerminalEvidenceResetCtx) ->
def loop_blocker_evidence_from_ctx(ctx: _LoopEvidenceCtx) -> LoopBlockerEvidence:
evidence = terminal_evidence_from_ctx(ctx)
+ # Older context snapshots may not carry fields added after the snapshot was created.
+ result_steer = getattr(ctx, "latest_evaluate_result_composition_steer", None)
return LoopBlockerEvidence(
outcome_gate_reason=evidence.outcome_gate_reason,
outcome_gate_workflow_run_id=evidence.outcome_gate_workflow_run_id,
@@ -246,6 +250,7 @@ def loop_blocker_evidence_from_ctx(ctx: _LoopEvidenceCtx) -> LoopBlockerEvidence
latest_workflow_run_id=evidence.latest_workflow_run_id,
anti_bot_blocked=evidence.anti_bot_blocked,
has_draft=evidence.has_draft,
+ latest_evaluate_result_composition_steer=result_steer,
)
@@ -365,6 +370,9 @@ _LOOP_BRANCH_COPY: dict[str, tuple[str, str]] = {
),
}
_LOOP_ANTI_BOT_BLOCKER_COPY = "The site's verification challenge was still keeping the submit/search control disabled."
+_LOOP_RESULT_COMPOSITION_BLOCKER_COPY = (
+ "I found loaded results on the page, but I got stuck before extracting the requested information."
+)
_LOOP_VERDICT_MAX_CHARS = 240
_LOOP_VERDICT_FAILED_PREFIX = "failed:"
# Fixed agent-advice tails appended by the recorded-reason producers; repair
@@ -490,6 +498,20 @@ def compose_loop_blocker_user_facing_reason(
if internal_reason_code == "loop_detected_credential_or_parameter_misconfig":
return _LOOP_CREDENTIAL_TEMPLATE, draft_tier
framing, ask = _LOOP_BRANCH_COPY.get(internal_reason_code or "", _LOOP_BRANCH_COPY["loop_detected_generic"])
+ result_steer = evidence.latest_evaluate_result_composition_steer if evidence is not None else None
+ if (
+ internal_reason_code == "loop_detected_consecutive_same_tool"
+ and blocked_tool == "evaluate"
+ and result_steer is not None
+ ):
+ # Loaded-result loops use their own framing while preserving the branch-specific ask.
+ user_facing, tiers = compose_terminal_evidence_user_facing_reason(
+ _LOOP_RESULT_COMPOSITION_BLOCKER_COPY,
+ ask,
+ evidence,
+ blocked_tool=blocked_tool,
+ )
+ return user_facing, ("loaded_results", *tiers)
return compose_terminal_evidence_user_facing_reason(framing, ask, evidence, blocked_tool=blocked_tool)
diff --git a/skyvern/forge/sdk/copilot/mcp_adapter.py b/skyvern/forge/sdk/copilot/mcp_adapter.py
index 5feda112d..81b95526c 100644
--- a/skyvern/forge/sdk/copilot/mcp_adapter.py
+++ b/skyvern/forge/sdk/copilot/mcp_adapter.py
@@ -66,6 +66,7 @@ _POST_HOOK_CONTEXT_ROLLBACK_FIELDS = (
"code_only_target_page_evidence_seen",
"last_evaluate_actionable_signature",
"last_evaluate_actionable_url",
+ "latest_evaluate_result_composition_steer",
"last_auto_acted_signature",
"reached_download_target",
"synthesized_block_offered",
diff --git a/skyvern/forge/sdk/copilot/result_evidence.py b/skyvern/forge/sdk/copilot/result_evidence.py
new file mode 100644
index 000000000..c0c23dbaf
--- /dev/null
+++ b/skyvern/forge/sdk/copilot/result_evidence.py
@@ -0,0 +1,138 @@
+"""Sanitized loaded-result evidence shared by Copilot steering and blockers."""
+
+from __future__ import annotations
+
+import re
+from collections.abc import Mapping
+from dataclasses import dataclass
+from typing import Any
+
+# Browser UI chrome, not Google Chrome. Keep this to generic empty-state and control text.
+_RESULT_CONTAINER_UI_CHROME_TOKENS = frozenset(
+ {
+ "and",
+ "are",
+ "bot",
+ "button",
+ "captcha",
+ "challenge",
+ "clear",
+ "control",
+ "disabled",
+ "enter",
+ "first",
+ "form",
+ "found",
+ "full",
+ "human",
+ "last",
+ "lookup",
+ "matching",
+ "name",
+ "record",
+ "records",
+ "reset",
+ "result",
+ "results",
+ "search",
+ "select",
+ "state",
+ "submit",
+ "verification",
+ "verify",
+ "you",
+ "your",
+ }
+)
+_EMPTY_RESULT_TEXT_PATTERNS = (
+ re.compile(r"\bno\s+(?:matching\s+)?(?:results?|records?)(?:\s+found)?\b"),
+ re.compile(r"\b0\s+(?:matching\s+)?(?:results?|records?)(?:\s+found)?\b"),
+)
+_TRANSIENT_RESULT_TEXT_PATTERNS = (
+ re.compile(r"(?:loading|please\s+wait|searching|fetching|processing)(?:\s+(?:results?|records?|items?))?\W*"),
+ re.compile(
+ r"(?:showing|displaying)\s+\d+\s*(?:-|to|through)\s*\d+\s+(?:of|/)\s+\d+(?:\s+(?:results?|records?|items?))?\W*"
+ ),
+)
+_MAX_MEANINGFUL_DATA_DEPTH = 10
+
+
+@dataclass(frozen=True)
+class LoadedResultCompositionEvidence:
+ result_container_count: int
+ table_result_container_count: int
+
+
+def _is_meaningful_data(value: Any, *, _depth: int = 0) -> bool:
+ if _depth > _MAX_MEANINGFUL_DATA_DEPTH:
+ return False
+ if value is None:
+ return False
+ if isinstance(value, (str, bytes)):
+ return bool(value)
+ if isinstance(value, Mapping):
+ return any(_is_meaningful_data(item, _depth=_depth + 1) for item in value.values())
+ if isinstance(value, (list, tuple, set)):
+ return any(_is_meaningful_data(item, _depth=_depth + 1) for item in value)
+ return True
+
+
+def _result_container_has_content(container: Mapping[str, Any]) -> bool:
+ row_count = container.get("row_count")
+ if isinstance(row_count, int) and row_count > 0:
+ return True
+ for key in ("sample_rows", "rows", "items"):
+ value = container.get(key)
+ if isinstance(value, list) and _is_meaningful_data(value):
+ return True
+ for key in ("content_excerpt", "sample_text", "text", "text_excerpt", "visible_results_evidence"):
+ value = container.get(key)
+ if isinstance(value, str):
+ if _text_has_non_ui_chrome_tokens(value):
+ return True
+ continue
+ if value:
+ return True
+ return False
+
+
+def _text_has_non_ui_chrome_tokens(value: str) -> bool:
+ normalized = value.strip().lower()
+ if any(pattern.search(normalized) for pattern in _EMPTY_RESULT_TEXT_PATTERNS):
+ return False
+ if any(pattern.fullmatch(normalized) for pattern in _TRANSIENT_RESULT_TEXT_PATTERNS):
+ return False
+ tokens = {token for token in re.findall(r"[a-z0-9]{2,}", normalized) if not token.isdigit()}
+ return bool(tokens - _RESULT_CONTAINER_UI_CHROME_TOKENS)
+
+
+def _result_container_is_table(container: Mapping[str, Any]) -> bool:
+ tag = container.get("tag")
+ return (
+ (isinstance(tag, str) and tag.lower() == "table")
+ or container.get("is_table") is True
+ or "row_selector" in container
+ )
+
+
+def loaded_result_composition_evidence_from_page(
+ evidence: Mapping[str, Any],
+) -> LoadedResultCompositionEvidence | None:
+ raw_containers = evidence.get("result_containers")
+ if not isinstance(raw_containers, list):
+ return None
+ containers = [container for container in raw_containers if isinstance(container, Mapping)]
+ meaningful = [container for container in containers if _result_container_has_content(container)]
+ if not meaningful:
+ return None
+ return LoadedResultCompositionEvidence(
+ result_container_count=len(meaningful),
+ table_result_container_count=sum(1 for container in meaningful if _result_container_is_table(container)),
+ )
+
+
+def loaded_result_composition_target_summary(evidence: LoadedResultCompositionEvidence) -> dict[str, int]:
+ return {
+ "result_container_count": evidence.result_container_count,
+ "table_result_container_count": evidence.table_result_container_count,
+ }
diff --git a/skyvern/forge/sdk/copilot/runtime.py b/skyvern/forge/sdk/copilot/runtime.py
index 3a338822e..9a7001264 100644
--- a/skyvern/forge/sdk/copilot/runtime.py
+++ b/skyvern/forge/sdk/copilot/runtime.py
@@ -38,6 +38,7 @@ if TYPE_CHECKING:
from skyvern.forge.sdk.copilot.completion_verification import CompletionVerificationResult
from skyvern.forge.sdk.copilot.reached_download_target import ReachedDownloadTarget
from skyvern.forge.sdk.copilot.request_policy import RequestPolicy
+ from skyvern.forge.sdk.copilot.result_evidence import LoadedResultCompositionEvidence
from skyvern.forge.sdk.copilot.run_outcome import RecordedRunOutcome
from skyvern.forge.sdk.copilot.turn_halt import TurnHalt
from skyvern.forge.sdk.routes.event_source_stream import EventSourceStream
@@ -263,6 +264,7 @@ class AgentContext:
post_run_current_page_inspection_workflow_run_id: str | None = None
last_evaluate_actionable_signature: str | None = None
last_evaluate_actionable_url: str | None = None
+ latest_evaluate_result_composition_steer: LoadedResultCompositionEvidence | None = None
last_auto_acted_signature: str | None = None
observed_browser_urls: list[str] = field(default_factory=list)
# Ephemeral within-turn scout captures; not persisted across turns.
diff --git a/skyvern/forge/sdk/copilot/tools/mcp_hooks.py b/skyvern/forge/sdk/copilot/tools/mcp_hooks.py
index eeaf36f30..469a108bc 100644
--- a/skyvern/forge/sdk/copilot/tools/mcp_hooks.py
+++ b/skyvern/forge/sdk/copilot/tools/mcp_hooks.py
@@ -49,6 +49,7 @@ from .scouting import (
_maybe_attach_reached_download_target,
_record_scouted_interaction,
_register_scout_interaction_observation,
+ _reset_evaluate_tracker,
_resolve_scout_role_name,
_steer_evaluate_result,
)
@@ -558,40 +559,43 @@ async def _evaluate_post_hook(
raw: dict[str, Any],
ctx: AgentContext,
) -> dict[str, Any]:
- if result.get("ok") and result.get("data"):
- _mark_page_inspected(ctx)
- result["data"].pop("sdk_equivalent", None)
- if "url" not in result["data"]:
- url, _ = await _resolve_url_title(raw, ctx)
- if url:
- result["data"]["url"] = url
- url = str(result["data"].get("url") or "")
- title = str(result["data"].get("title") or "")
- if not title:
- _, title = await _resolve_url_title(raw, ctx)
- observation_step = _record_composition_page_observation(
- ctx,
- source_tool="evaluate",
- url=url,
- title=title,
- observed_data=result["data"],
- append_to_flow=True,
- reached_via="auto",
- )
- if observation_step is not None:
- result["observation_step"] = observation_step
- result["data"]["observation_step"] = observation_step
- if _copilot_block_authoring_policy(
- ctx
- ) == BlockAuthoringPolicy.CODE_ONLY_BROWSER and _code_only_has_target_page_evidence(result["data"]):
- ctx.code_only_target_page_evidence_seen = True
- await _maybe_run_completion_verification_from_page_observation(
- ctx,
- url=url,
- title=title,
- observed_data=result["data"],
- )
- await _steer_evaluate_result(ctx, result, url=url)
+ data = result.get("data")
+ if not result.get("ok") or not isinstance(data, dict) or not data:
+ _reset_evaluate_tracker(ctx)
+ return result
+ _mark_page_inspected(ctx)
+ data.pop("sdk_equivalent", None)
+ if "url" not in data:
+ url, _ = await _resolve_url_title(raw, ctx)
+ if url:
+ data["url"] = url
+ url = str(data.get("url") or "")
+ title = str(data.get("title") or "")
+ if not title:
+ _, title = await _resolve_url_title(raw, ctx)
+ observation_step = _record_composition_page_observation(
+ ctx,
+ source_tool="evaluate",
+ url=url,
+ title=title,
+ observed_data=data,
+ append_to_flow=True,
+ reached_via="auto",
+ )
+ if observation_step is not None:
+ result["observation_step"] = observation_step
+ data["observation_step"] = observation_step
+ if _copilot_block_authoring_policy(
+ ctx
+ ) == BlockAuthoringPolicy.CODE_ONLY_BROWSER and _code_only_has_target_page_evidence(data):
+ ctx.code_only_target_page_evidence_seen = True
+ await _maybe_run_completion_verification_from_page_observation(
+ ctx,
+ url=url,
+ title=title,
+ observed_data=data,
+ )
+ await _steer_evaluate_result(ctx, result, url=url)
return result
diff --git a/skyvern/forge/sdk/copilot/tools/scouting.py b/skyvern/forge/sdk/copilot/tools/scouting.py
index 23b8bba09..9884ca0f1 100644
--- a/skyvern/forge/sdk/copilot/tools/scouting.py
+++ b/skyvern/forge/sdk/copilot/tools/scouting.py
@@ -28,6 +28,10 @@ from skyvern.forge.sdk.copilot.reached_download_target import (
derive_from_navigation_targets as _derive_reached_download_from_nav_targets,
)
from skyvern.forge.sdk.copilot.reached_download_target import guidance_for as _reached_download_guidance_for
+from skyvern.forge.sdk.copilot.result_evidence import (
+ loaded_result_composition_evidence_from_page,
+ loaded_result_composition_target_summary,
+)
from skyvern.forge.sdk.copilot.runtime import (
AgentContext,
PendingBrowserInteractionObservation,
@@ -221,7 +225,7 @@ def _record_scouted_interaction(
# press_key may be page-level, so it is recorded by key even with no selector; other tools require one.
if tool_name != "press_key" and not selector:
return
- _reset_evaluate_actionable_tracker(ctx)
+ _reset_evaluate_tracker(ctx)
artifact: ScoutedInteraction = {"tool_name": tool_name}
if selector:
artifact["selector"] = selector
@@ -472,11 +476,16 @@ _EVALUATE_ACTIONABLE_MAX_TARGETS = 4
_EVALUATE_ACTIONABLE_ACT_INSTRUCTION = (
"This page already exposes actionable targets; click the intended one rather than re-evaluating."
)
+_EVALUATE_RESULT_COMPOSITION_INSTRUCTION = (
+ "Loaded results are already visible on the current page; inspect this page for composition or author an "
+ "extraction/validation block from the loaded results instead of re-reading it."
+)
-def _reset_evaluate_actionable_tracker(ctx: AgentContext) -> None:
+def _reset_evaluate_tracker(ctx: AgentContext) -> None:
ctx.last_evaluate_actionable_signature = None
ctx.last_evaluate_actionable_url = None
+ ctx.latest_evaluate_result_composition_steer = None
def _actionable_target_identities(evidence: dict[str, Any]) -> list[tuple[str, str]]:
@@ -608,7 +617,15 @@ _EVALUATE_STEER_SHED_MARKER = "[omitted on repeat — act on the named target in
# Keys the steer must never shed: navigation/identity context plus the steer's own output.
_EVALUATE_STEER_ESSENTIAL_KEYS = frozenset(
- {"url", "title", "observation_step", "actionable_targets", "next_action", "next_action_reason"}
+ {
+ "url",
+ "title",
+ "observation_step",
+ "actionable_targets",
+ "composition_targets",
+ "next_action",
+ "next_action_reason",
+ }
)
# Nested bulky subfields inside an evaluate `result` dict (the raw page payload).
@@ -634,21 +651,23 @@ def _largest_non_essential_data_key(data: dict[str, Any]) -> str | None:
return candidates[0][0]
-def _fit_evaluate_steer_under_cap(result: dict[str, Any], data: dict[str, Any], *, is_repeat: bool) -> None:
+def _fit_evaluate_steer_under_cap(
+ result: dict[str, Any],
+ data: dict[str, Any],
+ *,
+ keep_raw_page_payload: bool,
+) -> None:
"""Keep the serialized result under the recent-output cap without ever head-slicing it.
- A first evaluate is reconnaissance: if naming targets alone pushes it over cap, drop only the
- advisory `actionable_targets` and leave the raw page (`data["result"]`) for the model to read.
- A repeat is an imperative steer: the model already saw the page on evaluate #1, so shed the bulky
- non-essential payload (the nested raw fields of `result`, then `result`/any large key wholesale,
- replaced by a short marker) while always keeping `next_action`, its reason, and >=1 target."""
+ Reconnaissance output needs the raw page payload available to the model; imperative steers have
+ enough structured evidence to shed bulky non-essential payload while always preserving the action."""
def over_cap() -> bool:
return len(json.dumps(result, default=str)) > _RECENT_TOOL_OUTPUT_CHAR_CAP
if not over_cap():
return
- if not is_repeat:
+ if keep_raw_page_payload:
data.pop("actionable_targets", None)
return
nested = data.get("result")
@@ -760,11 +779,30 @@ async def _maybe_steer_evaluate_to_action(
else page_evidence
)
if parsed is None or not has_bounded_page_schema(parsed):
- _reset_evaluate_actionable_tracker(ctx)
+ _reset_evaluate_tracker(ctx)
+ return False
+ loaded_results = loaded_result_composition_evidence_from_page(parsed)
+ if loaded_results is not None:
+ _reset_evaluate_tracker(ctx)
+ ctx.latest_evaluate_result_composition_steer = loaded_results
+ data.pop("actionable_targets", None)
+ data["composition_targets"] = loaded_result_composition_target_summary(loaded_results)
+ data["next_action"] = "compose_extraction"
+ data["next_action_reason"] = _EVALUATE_RESULT_COMPOSITION_INSTRUCTION
+ # The steer is structured enough to keep under cap without preserving raw page payload.
+ _fit_evaluate_steer_under_cap(result, data, keep_raw_page_payload=False)
+ LOG.info(
+ "copilot_evaluate_result_composition_steer",
+ url=url,
+ result_container_count=loaded_results.result_container_count,
+ table_result_container_count=loaded_results.table_result_container_count,
+ )
+ # The result is patched in-place; returning False keeps the normal tool-loop guard active.
return False
+ ctx.latest_evaluate_result_composition_steer = None
identities = _actionable_target_identities(parsed)
if not identities:
- _reset_evaluate_actionable_tracker(ctx)
+ _reset_evaluate_tracker(ctx)
return False
signature = _actionable_target_signature(identities)
# Strict full-URL match (fragment included): on an SPA a hash-route change
@@ -788,7 +826,7 @@ async def _maybe_steer_evaluate_to_action(
if is_repeat:
data["next_action"] = "click"
data["next_action_reason"] = _EVALUATE_ACTIONABLE_ACT_INSTRUCTION
- _fit_evaluate_steer_under_cap(result, data, is_repeat=is_repeat)
+ _fit_evaluate_steer_under_cap(result, data, keep_raw_page_payload=not is_repeat)
LOG.info(
"copilot_evaluate_actionable_target_steer",
url=url,
@@ -798,9 +836,10 @@ async def _maybe_steer_evaluate_to_action(
)
except Exception:
data.pop("actionable_targets", None)
+ data.pop("composition_targets", None)
data.pop("next_action", None)
data.pop("next_action_reason", None)
- _reset_evaluate_actionable_tracker(ctx)
+ _reset_evaluate_tracker(ctx)
LOG.warning("copilot_evaluate_actionable_target_steer_failed", exc_info=True)
return False
diff --git a/tests/unit/test_copilot_evaluate_page_evidence.py b/tests/unit/test_copilot_evaluate_page_evidence.py
index 4f095120a..ea1dfde87 100644
--- a/tests/unit/test_copilot_evaluate_page_evidence.py
+++ b/tests/unit/test_copilot_evaluate_page_evidence.py
@@ -7,6 +7,7 @@ import pytest
from skyvern.forge.sdk.copilot.build_phase import BuildPhase
from skyvern.forge.sdk.copilot.context import CopilotContext
from skyvern.forge.sdk.copilot.request_policy import RequestPolicy
+from skyvern.forge.sdk.copilot.result_evidence import LoadedResultCompositionEvidence
from skyvern.forge.sdk.copilot.tools import (
_evaluate_post_hook,
_inspect_page_for_composition_impl,
@@ -33,6 +34,33 @@ def _ctx() -> CopilotContext:
)
+ "result",
+ [
+ {"ok": False, "error": "evaluate failed"},
+ {"ok": True},
+ {"ok": True, "data": {}},
+ {"ok": True, "data": []},
+ ],
+)
+async def test_evaluate_post_hook_resets_steer_on_unusable_result(result: dict[str, object]) -> None:
+ ctx = _ctx()
+ ctx.last_evaluate_actionable_signature = "stale-signature"
+ ctx.last_evaluate_actionable_url = "https://example.test/old"
+ ctx.latest_evaluate_result_composition_steer = LoadedResultCompositionEvidence(
+ result_container_count=1,
+ table_result_container_count=1,
+ )
+
+ updated = await _evaluate_post_hook(result, raw={}, ctx=ctx)
+
+ assert updated is result
+ assert ctx.last_evaluate_actionable_signature is None
+ assert ctx.last_evaluate_actionable_url is None
+ assert ctx.latest_evaluate_result_composition_steer is None
+
+
@pytest.mark.asyncio
async def test_evaluate_nested_rows_records_interaction_observation_step() -> None:
ctx = _ctx()
diff --git a/tests/unit/test_copilot_hooks.py b/tests/unit/test_copilot_hooks.py
index 4675ce706..d808bdc9b 100644
--- a/tests/unit/test_copilot_hooks.py
+++ b/tests/unit/test_copilot_hooks.py
@@ -341,6 +341,27 @@ class TestSchemaOverlay:
class TestMCPFailedStepLoopDetection:
+ def test_post_hook_context_snapshot_restores_loaded_result_composition_steer(self) -> None:
+ from skyvern.forge.sdk.copilot.mcp_adapter import _restore_post_hook_context, _snapshot_post_hook_context
+ from skyvern.forge.sdk.copilot.result_evidence import LoadedResultCompositionEvidence
+
+ initial_steer = LoadedResultCompositionEvidence(
+ result_container_count=1,
+ table_result_container_count=1,
+ )
+ ctx = SimpleNamespace(
+ latest_evaluate_result_composition_steer=initial_steer,
+ )
+
+ snapshot = _snapshot_post_hook_context(ctx)
+ ctx.latest_evaluate_result_composition_steer = LoadedResultCompositionEvidence(
+ result_container_count=2,
+ table_result_container_count=0,
+ )
+ _restore_post_hook_context(ctx, snapshot)
+
+ assert ctx.latest_evaluate_result_composition_steer == initial_steer
+
@pytest.mark.asyncio
async def test_post_hook_exception_preserves_successful_tool_result(self, monkeypatch: pytest.MonkeyPatch) -> None:
from skyvern.forge.sdk.copilot import mcp_adapter
diff --git a/tests/unit/test_copilot_loop_detection_blocker.py b/tests/unit/test_copilot_loop_detection_blocker.py
index ab4210067..7ad15ac09 100644
--- a/tests/unit/test_copilot_loop_detection_blocker.py
+++ b/tests/unit/test_copilot_loop_detection_blocker.py
@@ -17,6 +17,7 @@ from skyvern.forge.sdk.copilot.blocker_signal import (
from skyvern.forge.sdk.copilot.context import CopilotContext
from skyvern.forge.sdk.copilot.mcp_adapter import SchemaOverlay, SkyvernOverlayMCPServer, _stash_and_emit_loop_blocker
from skyvern.forge.sdk.copilot.output_policy import CopilotOutputKind, evaluate_output_policy
+from skyvern.forge.sdk.copilot.result_evidence import LoadedResultCompositionEvidence
from skyvern.forge.sdk.copilot.run_outcome import TERMINAL_CHALLENGE_BLOCKER_REASON_CODE, RecordedRunOutcome
from skyvern.forge.sdk.copilot.tools import _build_loop_blocker_signal, _tool_loop_error
from skyvern.forge.sdk.copilot.tools.mcp_hooks import get_skyvern_mcp_alias_map
@@ -137,6 +138,60 @@ def test_native_dispatch_consecutive_tool_loop_sets_signal() -> None:
assert ctx.turn_halt.kind == TurnHaltKind.LOOP_DETECTED
+def test_consecutive_evaluate_loop_with_loaded_results_uses_goal_aware_copy() -> None:
+ ctx = _ctx(consecutive_tool_tracker=["evaluate", "evaluate"])
+ ctx.latest_evaluate_result_composition_steer = LoadedResultCompositionEvidence(
+ result_container_count=1,
+ table_result_container_count=1,
+ )
+ payload = _tool_loop_error(ctx, "evaluate", None)
+
+ assert payload is not None
+ assert payload.startswith("LOOP DETECTED: 'evaluate' has been called 3 times consecutively.")
+ assert isinstance(ctx.blocker_signal, CopilotToolBlockerSignal)
+ assert ctx.blocker_signal.internal_reason_code == "loop_detected_consecutive_same_tool"
+ assert "loaded results" in ctx.blocker_signal.user_facing_reason
+ assert "extracting the requested information" in ctx.blocker_signal.user_facing_reason
+ assert "retrying the same step" not in ctx.blocker_signal.user_facing_reason
+ assert dict(ctx.blocker_signal.extra) == {"loop_evidence_tiers": ["loaded_results"]}
+ assert_clean_user_facing_text(ctx.blocker_signal.user_facing_reason, blocked_tool="evaluate")
+
+
+def test_consecutive_non_evaluate_loop_ignores_loaded_result_steer() -> None:
+ ctx = _ctx(consecutive_tool_tracker=["list_credentials", "list_credentials"])
+ ctx.latest_evaluate_result_composition_steer = LoadedResultCompositionEvidence(
+ result_container_count=1,
+ table_result_container_count=1,
+ )
+ payload = _tool_loop_error(ctx, "list_credentials", None)
+
+ assert payload is not None
+ assert isinstance(ctx.blocker_signal, CopilotToolBlockerSignal)
+ assert ctx.blocker_signal.internal_reason_code == "loop_detected_consecutive_same_tool"
+ assert "loaded results" not in ctx.blocker_signal.user_facing_reason
+ assert ctx.blocker_signal.user_facing_reason == (
+ "I'm stuck retrying the same step. Tell me what to change and I'll try a different approach."
+ )
+ assert dict(ctx.blocker_signal.extra) == {}
+
+
+def test_consecutive_evaluate_loop_ignores_stale_composition_page_results_without_latest_steer() -> None:
+ ctx = _ctx(consecutive_tool_tracker=["evaluate", "evaluate"])
+ ctx.composition_page_evidence = {
+ "result_containers": [{"tag": "table", "selector": "#old-results", "row_count": 3}],
+ }
+ payload = _tool_loop_error(ctx, "evaluate", None)
+
+ assert payload is not None
+ assert isinstance(ctx.blocker_signal, CopilotToolBlockerSignal)
+ assert ctx.blocker_signal.internal_reason_code == "loop_detected_consecutive_same_tool"
+ assert ctx.blocker_signal.user_facing_reason == (
+ "I'm stuck retrying the same step. Tell me what to change and I'll try a different approach."
+ )
+ assert dict(ctx.blocker_signal.extra) == {}
+ assert_clean_user_facing_text(ctx.blocker_signal.user_facing_reason, blocked_tool="evaluate")
+
+
def test_native_and_mcp_paths_produce_equivalent_loop_signal() -> None:
"""Native dispatch and the MCP adapter must produce the same signal shape
for the same loop-detection message so a regression in one path can't
diff --git a/tests/unit/test_copilot_result_evidence.py b/tests/unit/test_copilot_result_evidence.py
new file mode 100644
index 000000000..911f9e938
--- /dev/null
+++ b/tests/unit/test_copilot_result_evidence.py
@@ -0,0 +1,77 @@
+from __future__ import annotations
+
+from skyvern.forge.sdk.copilot.result_evidence import loaded_result_composition_evidence_from_page
+
+
+def test_loaded_result_evidence_counts_populated_tables() -> None:
+ evidence = loaded_result_composition_evidence_from_page(
+ {
+ "result_containers": [
+ {"tag": "table", "selector": "#results", "row_count": 2},
+ {"selector": "#cards", "sample_rows": ["May 2026 statement available"]},
+ ],
+ }
+ )
+
+ assert evidence is not None
+ assert evidence.result_container_count == 2
+ assert evidence.table_result_container_count == 1
+
+
+def test_caption_only_result_container_is_not_composition_evidence() -> None:
+ evidence = loaded_result_composition_evidence_from_page(
+ {
+ "result_containers": [
+ {"selector": "#count", "text": "12 matching records"},
+ {"selector": "#results", "text": "3 matching records"},
+ {"selector": "#none", "text": "No results found"},
+ {"selector": "#empty", "text": "0 results found"},
+ ],
+ }
+ )
+
+ assert evidence is None
+
+
+def test_result_container_text_uses_non_ui_chrome_tokens() -> None:
+ evidence = loaded_result_composition_evidence_from_page(
+ {"result_containers": [{"selector": "#results", "text": "Statement May 2026 available"}]}
+ )
+
+ assert evidence is not None
+ assert evidence.result_container_count == 1
+
+
+def test_standalone_no_can_be_result_payload_text() -> None:
+ evidence = loaded_result_composition_evidence_from_page(
+ {"result_containers": [{"selector": "#results", "text": "No"}]}
+ )
+
+ assert evidence is not None
+ assert evidence.result_container_count == 1
+
+
+def test_transient_status_result_container_is_not_composition_evidence() -> None:
+ evidence = loaded_result_composition_evidence_from_page(
+ {
+ "result_containers": [
+ {"selector": "#loading", "text": "Loading..."},
+ {"selector": "#wait", "text": "Please wait"},
+ {"selector": "#pagination", "text": "Showing 1-20 of 100"},
+ ],
+ }
+ )
+
+ assert evidence is None
+
+
+def test_deep_result_container_payload_does_not_recurse_unbounded() -> None:
+ nested: object = "value"
+ for _ in range(20):
+ nested = [nested]
+
+ evidence = loaded_result_composition_evidence_from_page(
+ {"result_containers": [{"selector": "#results", "sample_rows": [nested]}]}
+ )
+
+ assert evidence is None
diff --git a/tests/unit/test_evaluate_actionable_target.py b/tests/unit/test_evaluate_actionable_target.py
index 447ea47d3..a36ea0392 100644
--- a/tests/unit/test_evaluate_actionable_target.py
+++ b/tests/unit/test_evaluate_actionable_target.py
@@ -33,6 +33,36 @@ def _full_packet() -> dict:
}
+def _result_table_packet() -> dict:
+ return {
+ "forms": [],
+ "navigation_targets": [{"selector": "#details", "text": "Details", "href": "/details"}],
+ "result_containers": [
+ {
+ "tag": "table",
+ "selector": "#results",
+ "text": "Results",
+ "row_count": 2,
+ "sample_rows": ["Result A May 2026 $42.00", "Result B May 2026 $51.00"],
+ }
+ ],
+ }
+
+
+def _row_result_packet() -> dict:
+ return {
+ "forms": [],
+ "navigation_targets": [],
+ "result_containers": [
+ {
+ "selector": "#results",
+ "row_count": 1,
+ "sample_rows": ["May 2026 statement available"],
+ }
+ ],
+ }
+
+
def _ctx() -> AgentContext:
return AgentContext.__new__(AgentContext)
@@ -76,6 +106,60 @@ async def test_first_evaluate_names_targets_without_next_action(monkeypatch) ->
assert "next_action" not in result["data"]
+async def test_first_loaded_result_table_steers_to_composition_not_click_even_with_actionable_targets(
+ monkeypatch,
+) -> None:
+ async def fake_evidence(ctx, *, url):
+ return _result_table_packet()
+
+ monkeypatch.setattr(scouting, "_scout_act_observe_page_evidence", fake_evidence)
+ ctx = _ctx()
+ ctx.last_evaluate_actionable_signature = "stale"
+ ctx.last_evaluate_actionable_url = "https://example.com/previous"
+ result = {
+ "ok": True,
+ "data": {
+ "url": "https://example.com/results",
+ "actionable_targets": [{"selector": "#details", "text": "Details"}],
+ },
+ }
+
+ await scouting._maybe_steer_evaluate_to_action(ctx, result, url="https://example.com/results")
+
+ assert result["data"].get("next_action") == "compose_extraction"
+ assert "extract" in result["data"]["next_action_reason"]
+ assert "click" not in result["data"]["next_action_reason"].lower()
+ assert "actionable_targets" not in result["data"]
+ assert result["data"]["composition_targets"] == {
+ "result_container_count": 1,
+ "table_result_container_count": 1,
+ }
+ assert ctx.latest_evaluate_result_composition_steer.result_container_count == 1
+ assert ctx.last_evaluate_actionable_signature is None
+ assert ctx.last_evaluate_actionable_url is None
+
+
+async def test_first_loaded_result_rows_without_table_marker_steers_to_composition(monkeypatch) -> None:
+ async def fake_evidence(ctx, *, url):
+ return _row_result_packet()
+
+ monkeypatch.setattr(scouting, "_scout_act_observe_page_evidence", fake_evidence)
+ ctx = _ctx()
+ ctx.last_evaluate_actionable_signature = None
+ ctx.last_evaluate_actionable_url = None
+ result = {"ok": True, "data": {"url": "https://example.com/results"}}
+
+ await scouting._maybe_steer_evaluate_to_action(ctx, result, url="https://example.com/results")
+
+ assert result["data"].get("next_action") == "compose_extraction"
+ assert result["data"]["composition_targets"] == {
+ "result_container_count": 1,
+ "table_result_container_count": 0,
+ }
+
+
@pytest.mark.asyncio
async def test_repeat_evaluate_same_page_steers_to_click(monkeypatch) -> None:
async def fake_evidence(ctx, *, url):
@@ -162,6 +246,28 @@ async def test_repeat_scalar_result_sheds_under_cap(monkeypatch) -> None:
assert '"next_action"' in serialized[: scouting._RECENT_TOOL_OUTPUT_CHAR_CAP]
+async def test_first_loaded_result_table_sheds_large_payload_but_keeps_composition_steer(monkeypatch) -> None:
+ ctx = _ctx()
+ ctx.last_evaluate_actionable_signature = None
+ ctx.last_evaluate_actionable_url = None
+ url = "https://app.example.com/results"
+
+ async def fake(c, *, url):
+ return _result_table_packet()
+
+ monkeypatch.setattr(scouting, "_scout_act_observe_page_evidence", fake)
+ big = "x" * 10000
+ result = {"ok": True, "data": {"url": url, "result": {"html": big, "text": big}}}
+ await scouting._maybe_steer_evaluate_to_action(ctx, result, url=url)
+ serialized = json.dumps(result, default=str)
+ assert result["data"].get("next_action") == "compose_extraction"
+ assert result["data"].get("composition_targets")
+ assert len(serialized) <= scouting._RECENT_TOOL_OUTPUT_CHAR_CAP
+ assert '"next_action"' in serialized[: scouting._RECENT_TOOL_OUTPUT_CHAR_CAP]
+ assert "compose_extraction" in serialized
+
+
@pytest.mark.asyncio
async def test_repeat_nested_result_sheds_under_cap(monkeypatch) -> None:
ctx = _ctx()
@@ -256,8 +362,9 @@ async def test_probe_none_leaves_data_untouched_and_resets(monkeypatch) -> None:
@pytest.mark.asyncio
async def test_probe_empty_collections_no_steer(monkeypatch) -> None:
ctx = _ctx()
- ctx.last_evaluate_actionable_signature = None
+ ctx.last_evaluate_actionable_signature = "seed"
ctx.last_evaluate_actionable_url = None
+ ctx.latest_evaluate_result_composition_steer = object()
async def fake(c, *, url):
return {"forms": [], "navigation_targets": [], "result_containers": []}
@@ -266,6 +373,7 @@ async def test_probe_empty_collections_no_steer(monkeypatch) -> None:
result = {"ok": True, "data": {"url": "https://app.example.com/x"}}
await scouting._maybe_steer_evaluate_to_action(ctx, result, url="https://app.example.com/x")
assert "actionable_targets" not in result["data"]
+ assert ctx.latest_evaluate_result_composition_steer is None
@pytest.mark.asyncio