Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
SKY-11182: steer loaded results into composition (#6723)
Andrew Neilson9d6c5f11f2ca
diff --git a/skyvern/forge/sdk/copilot/blocker_signal.py b/skyvern/forge/sdk/copilot/blocker_signal.pyindex 1f9978c3a..d2a460844 100644--- a/skyvern/forge/sdk/copilot/blocker_signal.py+++ b/skyvern/forge/sdk/copilot/blocker_signal.py@@ -12,6 +12,7 @@ import structlogfrom pydantic import BaseModel, ConfigDict, Field, field_serializer, model_validatorfrom skyvern.forge.sdk.copilot.failure_tracking import ACTIVE_RUN_TERMINAL_EVIDENCE_REASON_CODE+from skyvern.forge.sdk.copilot.result_evidence import LoadedResultCompositionEvidencefrom skyvern.forge.sdk.copilot.run_outcome import RecordedRunOutcomeBlockerKind = Literal[@@ -171,7 +172,7 @@ class TerminalEvidence:@dataclass(frozen=True)class LoopBlockerEvidence(TerminalEvidence):- pass+ latest_evaluate_result_composition_steer: LoadedResultCompositionEvidence | None = Noneclass _LoopEvidenceCtx(Protocol):@@ -183,6 +184,7 @@ class _LoopEvidenceCtx(Protocol):staged_workflow: Any | Nonestaged_workflow_yaml: str | Nonehas_staged_proposal: bool+ latest_evaluate_result_composition_steer: LoadedResultCompositionEvidence | Noneclass _BlockerSignalCtx(_LoopEvidenceCtx, Protocol):@@ -239,6 +241,8 @@ def clear_terminal_evidence_on_workflow_edit(ctx: _TerminalEvidenceResetCtx) ->def loop_blocker_evidence_from_ctx(ctx: _LoopEvidenceCtx) -> LoopBlockerEvidence:evidence = terminal_evidence_from_ctx(ctx)+ # Older context snapshots may not carry fields added after the snapshot was created.+ result_steer = getattr(ctx, "latest_evaluate_result_composition_steer", None)return LoopBlockerEvidence(outcome_gate_reason=evidence.outcome_gate_reason,outcome_gate_workflow_run_id=evidence.outcome_gate_workflow_run_id,@@ -246,6 +250,7 @@ def loop_blocker_evidence_from_ctx(ctx: _LoopEvidenceCtx) -> LoopBlockerEvidencelatest_workflow_run_id=evidence.latest_workflow_run_id,anti_bot_blocked=evidence.anti_bot_blocked,has_draft=evidence.has_draft,+ latest_evaluate_result_composition_steer=result_steer,)@@ -365,6 +370,9 @@ _LOOP_BRANCH_COPY: dict[str, tuple[str, str]] = {),}_LOOP_ANTI_BOT_BLOCKER_COPY = "The site's verification challenge was still keeping the submit/search control disabled."+_LOOP_RESULT_COMPOSITION_BLOCKER_COPY = (+ "I found loaded results on the page, but I got stuck before extracting the requested information."+)_LOOP_VERDICT_MAX_CHARS = 240_LOOP_VERDICT_FAILED_PREFIX = "failed:"# Fixed agent-advice tails appended by the recorded-reason producers; repair@@ -490,6 +498,20 @@ def compose_loop_blocker_user_facing_reason(if internal_reason_code == "loop_detected_credential_or_parameter_misconfig":return _LOOP_CREDENTIAL_TEMPLATE, draft_tierframing, ask = _LOOP_BRANCH_COPY.get(internal_reason_code or "", _LOOP_BRANCH_COPY["loop_detected_generic"])+ result_steer = evidence.latest_evaluate_result_composition_steer if evidence is not None else None+ if (+ internal_reason_code == "loop_detected_consecutive_same_tool"+ and blocked_tool == "evaluate"+ and result_steer is not None+ ):+ # Loaded-result loops use their own framing while preserving the branch-specific ask.+ user_facing, tiers = compose_terminal_evidence_user_facing_reason(+ _LOOP_RESULT_COMPOSITION_BLOCKER_COPY,+ ask,+ evidence,+ blocked_tool=blocked_tool,+ )+ return user_facing, ("loaded_results", *tiers)return compose_terminal_evidence_user_facing_reason(framing, ask, evidence, blocked_tool=blocked_tool)diff --git a/skyvern/forge/sdk/copilot/mcp_adapter.py b/skyvern/forge/sdk/copilot/mcp_adapter.pyindex 5feda112d..81b95526c 100644--- a/skyvern/forge/sdk/copilot/mcp_adapter.py+++ b/skyvern/forge/sdk/copilot/mcp_adapter.py@@ -66,6 +66,7 @@ _POST_HOOK_CONTEXT_ROLLBACK_FIELDS = ("code_only_target_page_evidence_seen","last_evaluate_actionable_signature","last_evaluate_actionable_url",+ "latest_evaluate_result_composition_steer","last_auto_acted_signature","reached_download_target","synthesized_block_offered",diff --git a/skyvern/forge/sdk/copilot/result_evidence.py b/skyvern/forge/sdk/copilot/result_evidence.pynew file mode 100644index 000000000..c0c23dbaf--- /dev/null+++ b/skyvern/forge/sdk/copilot/result_evidence.py@@ -0,0 +1,138 @@+"""Sanitized loaded-result evidence shared by Copilot steering and blockers."""++from __future__ import annotations++import re+from collections.abc import Mapping+from dataclasses import dataclass+from typing import Any++# Browser UI chrome, not Google Chrome. Keep this to generic empty-state and control text.+_RESULT_CONTAINER_UI_CHROME_TOKENS = frozenset(+ {+ "and",+ "are",+ "bot",+ "button",+ "captcha",+ "challenge",+ "clear",+ "control",+ "disabled",+ "enter",+ "first",+ "form",+ "found",+ "full",+ "human",+ "last",+ "lookup",+ "matching",+ "name",+ "record",+ "records",+ "reset",+ "result",+ "results",+ "search",+ "select",+ "state",+ "submit",+ "verification",+ "verify",+ "you",+ "your",+ }+)+_EMPTY_RESULT_TEXT_PATTERNS = (+ re.compile(r"\bno\s+(?:matching\s+)?(?:results?|records?)(?:\s+found)?\b"),+ re.compile(r"\b0\s+(?:matching\s+)?(?:results?|records?)(?:\s+found)?\b"),+)+_TRANSIENT_RESULT_TEXT_PATTERNS = (+ re.compile(r"(?:loading|please\s+wait|searching|fetching|processing)(?:\s+(?:results?|records?|items?))?\W*"),+ re.compile(+ r"(?:showing|displaying)\s+\d+\s*(?:-|to|through)\s*\d+\s+(?:of|/)\s+\d+(?:\s+(?:results?|records?|items?))?\W*"+ ),+)+_MAX_MEANINGFUL_DATA_DEPTH = 10+++@dataclass(frozen=True)+class LoadedResultCompositionEvidence:+ result_container_count: int+ table_result_container_count: int+++def _is_meaningful_data(value: Any, *, _depth: int = 0) -> bool:+ if _depth > _MAX_MEANINGFUL_DATA_DEPTH:+ return False+ if value is None:+ return False+ if isinstance(value, (str, bytes)):+ return bool(value)+ if isinstance(value, Mapping):+ return any(_is_meaningful_data(item, _depth=_depth + 1) for item in value.values())+ if isinstance(value, (list, tuple, set)):+ return any(_is_meaningful_data(item, _depth=_depth + 1) for item in value)+ return True+++def _result_container_has_content(container: Mapping[str, Any]) -> bool:+ row_count = container.get("row_count")+ if isinstance(row_count, int) and row_count > 0:+ return True+ for key in ("sample_rows", "rows", "items"):+ value = container.get(key)+ if isinstance(value, list) and _is_meaningful_data(value):+ return True+ for key in ("content_excerpt", "sample_text", "text", "text_excerpt", "visible_results_evidence"):+ value = container.get(key)+ if isinstance(value, str):+ if _text_has_non_ui_chrome_tokens(value):+ return True+ continue+ if value:+ return True+ return False+++def _text_has_non_ui_chrome_tokens(value: str) -> bool:+ normalized = value.strip().lower()+ if any(pattern.search(normalized) for pattern in _EMPTY_RESULT_TEXT_PATTERNS):+ return False+ if any(pattern.fullmatch(normalized) for pattern in _TRANSIENT_RESULT_TEXT_PATTERNS):+ return False+ tokens = {token for token in re.findall(r"[a-z0-9]{2,}", normalized) if not token.isdigit()}+ return bool(tokens - _RESULT_CONTAINER_UI_CHROME_TOKENS)+++def _result_container_is_table(container: Mapping[str, Any]) -> bool:+ tag = container.get("tag")+ return (+ (isinstance(tag, str) and tag.lower() == "table")+ or container.get("is_table") is True+ or "row_selector" in container+ )+++def loaded_result_composition_evidence_from_page(+ evidence: Mapping[str, Any],+) -> LoadedResultCompositionEvidence | None:+ raw_containers = evidence.get("result_containers")+ if not isinstance(raw_containers, list):+ return None+ containers = [container for container in raw_containers if isinstance(container, Mapping)]+ meaningful = [container for container in containers if _result_container_has_content(container)]+ if not meaningful:+ return None+ return LoadedResultCompositionEvidence(+ result_container_count=len(meaningful),+ table_result_container_count=sum(1 for container in meaningful if _result_container_is_table(container)),+ )+++def loaded_result_composition_target_summary(evidence: LoadedResultCompositionEvidence) -> dict[str, int]:+ return {+ "result_container_count": evidence.result_container_count,+ "table_result_container_count": evidence.table_result_container_count,+ }diff --git a/skyvern/forge/sdk/copilot/runtime.py b/skyvern/forge/sdk/copilot/runtime.pyindex 3a338822e..9a7001264 100644--- a/skyvern/forge/sdk/copilot/runtime.py+++ b/skyvern/forge/sdk/copilot/runtime.py@@ -38,6 +38,7 @@ if TYPE_CHECKING:from skyvern.forge.sdk.copilot.completion_verification import CompletionVerificationResultfrom skyvern.forge.sdk.copilot.reached_download_target import ReachedDownloadTargetfrom skyvern.forge.sdk.copilot.request_policy import RequestPolicy+ from skyvern.forge.sdk.copilot.result_evidence import LoadedResultCompositionEvidencefrom skyvern.forge.sdk.copilot.run_outcome import RecordedRunOutcomefrom skyvern.forge.sdk.copilot.turn_halt import TurnHaltfrom skyvern.forge.sdk.routes.event_source_stream import EventSourceStream@@ -263,6 +264,7 @@ class AgentContext:post_run_current_page_inspection_workflow_run_id: str | None = Nonelast_evaluate_actionable_signature: str | None = Nonelast_evaluate_actionable_url: str | None = None+ latest_evaluate_result_composition_steer: LoadedResultCompositionEvidence | None = Nonelast_auto_acted_signature: str | None = Noneobserved_browser_urls: list[str] = field(default_factory=list)# Ephemeral within-turn scout captures; not persisted across turns.diff --git a/skyvern/forge/sdk/copilot/tools/mcp_hooks.py b/skyvern/forge/sdk/copilot/tools/mcp_hooks.pyindex eeaf36f30..469a108bc 100644--- a/skyvern/forge/sdk/copilot/tools/mcp_hooks.py+++ b/skyvern/forge/sdk/copilot/tools/mcp_hooks.py@@ -49,6 +49,7 @@ from .scouting import (_maybe_attach_reached_download_target,_record_scouted_interaction,_register_scout_interaction_observation,+ _reset_evaluate_tracker,_resolve_scout_role_name,_steer_evaluate_result,)@@ -558,40 +559,43 @@ async def _evaluate_post_hook(raw: dict[str, Any],ctx: AgentContext,) -> dict[str, Any]:- if result.get("ok") and result.get("data"):- _mark_page_inspected(ctx)- result["data"].pop("sdk_equivalent", None)- if "url" not in result["data"]:- url, _ = await _resolve_url_title(raw, ctx)- if url:- result["data"]["url"] = url- url = str(result["data"].get("url") or "")- title = str(result["data"].get("title") or "")- if not title:- _, title = await _resolve_url_title(raw, ctx)- observation_step = _record_composition_page_observation(- ctx,- source_tool="evaluate",- url=url,- title=title,- observed_data=result["data"],- append_to_flow=True,- reached_via="auto",- )- if observation_step is not None:- result["observation_step"] = observation_step- result["data"]["observation_step"] = observation_step- if _copilot_block_authoring_policy(- ctx- ) == BlockAuthoringPolicy.CODE_ONLY_BROWSER and _code_only_has_target_page_evidence(result["data"]):- ctx.code_only_target_page_evidence_seen = True- await _maybe_run_completion_verification_from_page_observation(- ctx,- url=url,- title=title,- observed_data=result["data"],- )- await _steer_evaluate_result(ctx, result, url=url)+ data = result.get("data")+ if not result.get("ok") or not isinstance(data, dict) or not data:+ _reset_evaluate_tracker(ctx)+ return result+ _mark_page_inspected(ctx)+ data.pop("sdk_equivalent", None)+ if "url" not in data:+ url, _ = await _resolve_url_title(raw, ctx)+ if url:+ data["url"] = url+ url = str(data.get("url") or "")+ title = str(data.get("title") or "")+ if not title:+ _, title = await _resolve_url_title(raw, ctx)+ observation_step = _record_composition_page_observation(+ ctx,+ source_tool="evaluate",+ url=url,+ title=title,+ observed_data=data,+ append_to_flow=True,+ reached_via="auto",+ )+ if observation_step is not None:+ result["observation_step"] = observation_step+ data["observation_step"] = observation_step+ if _copilot_block_authoring_policy(+ ctx+ ) == BlockAuthoringPolicy.CODE_ONLY_BROWSER and _code_only_has_target_page_evidence(data):+ ctx.code_only_target_page_evidence_seen = True+ await _maybe_run_completion_verification_from_page_observation(+ ctx,+ url=url,+ title=title,+ observed_data=data,+ )+ await _steer_evaluate_result(ctx, result, url=url)return resultdiff --git a/skyvern/forge/sdk/copilot/tools/scouting.py b/skyvern/forge/sdk/copilot/tools/scouting.pyindex 23b8bba09..9884ca0f1 100644--- a/skyvern/forge/sdk/copilot/tools/scouting.py+++ b/skyvern/forge/sdk/copilot/tools/scouting.py@@ -28,6 +28,10 @@ from skyvern.forge.sdk.copilot.reached_download_target import (derive_from_navigation_targets as _derive_reached_download_from_nav_targets,)from skyvern.forge.sdk.copilot.reached_download_target import guidance_for as _reached_download_guidance_for+from skyvern.forge.sdk.copilot.result_evidence import (+ loaded_result_composition_evidence_from_page,+ loaded_result_composition_target_summary,+)from skyvern.forge.sdk.copilot.runtime import (AgentContext,PendingBrowserInteractionObservation,@@ -221,7 +225,7 @@ def _record_scouted_interaction(# press_key may be page-level, so it is recorded by key even with no selector; other tools require one.if tool_name != "press_key" and not selector:return- _reset_evaluate_actionable_tracker(ctx)+ _reset_evaluate_tracker(ctx)artifact: ScoutedInteraction = {"tool_name": tool_name}if selector:artifact["selector"] = selector@@ -472,11 +476,16 @@ _EVALUATE_ACTIONABLE_MAX_TARGETS = 4_EVALUATE_ACTIONABLE_ACT_INSTRUCTION = ("This page already exposes actionable targets; click the intended one rather than re-evaluating.")+_EVALUATE_RESULT_COMPOSITION_INSTRUCTION = (+ "Loaded results are already visible on the current page; inspect this page for composition or author an "+ "extraction/validation block from the loaded results instead of re-reading it."+)-def _reset_evaluate_actionable_tracker(ctx: AgentContext) -> None:+def _reset_evaluate_tracker(ctx: AgentContext) -> None:ctx.last_evaluate_actionable_signature = Nonectx.last_evaluate_actionable_url = None+ ctx.latest_evaluate_result_composition_steer = Nonedef _actionable_target_identities(evidence: dict[str, Any]) -> list[tuple[str, str]]:@@ -608,7 +617,15 @@ _EVALUATE_STEER_SHED_MARKER = "[omitted on repeat — act on the named target in# Keys the steer must never shed: navigation/identity context plus the steer's own output._EVALUATE_STEER_ESSENTIAL_KEYS = frozenset(- {"url", "title", "observation_step", "actionable_targets", "next_action", "next_action_reason"}+ {+ "url",+ "title",+ "observation_step",+ "actionable_targets",+ "composition_targets",+ "next_action",+ "next_action_reason",+ })# Nested bulky subfields inside an evaluate `result` dict (the raw page payload).@@ -634,21 +651,23 @@ def _largest_non_essential_data_key(data: dict[str, Any]) -> str | None:return candidates[0][0]-def _fit_evaluate_steer_under_cap(result: dict[str, Any], data: dict[str, Any], *, is_repeat: bool) -> None:+def _fit_evaluate_steer_under_cap(+ result: dict[str, Any],+ data: dict[str, Any],+ *,+ keep_raw_page_payload: bool,+) -> None:"""Keep the serialized result under the recent-output cap without ever head-slicing it.- A first evaluate is reconnaissance: if naming targets alone pushes it over cap, drop only the- advisory `actionable_targets` and leave the raw page (`data["result"]`) for the model to read.- A repeat is an imperative steer: the model already saw the page on evaluate #1, so shed the bulky- non-essential payload (the nested raw fields of `result`, then `result`/any large key wholesale,- replaced by a short marker) while always keeping `next_action`, its reason, and >=1 target."""+ Reconnaissance output needs the raw page payload available to the model; imperative steers have+ enough structured evidence to shed bulky non-essential payload while always preserving the action."""def over_cap() -> bool:return len(json.dumps(result, default=str)) > _RECENT_TOOL_OUTPUT_CHAR_CAPif not over_cap():return- if not is_repeat:+ if keep_raw_page_payload:data.pop("actionable_targets", None)returnnested = data.get("result")@@ -760,11 +779,30 @@ async def _maybe_steer_evaluate_to_action(else page_evidence)if parsed is None or not has_bounded_page_schema(parsed):- _reset_evaluate_actionable_tracker(ctx)+ _reset_evaluate_tracker(ctx)+ return False+ loaded_results = loaded_result_composition_evidence_from_page(parsed)+ if loaded_results is not None:+ _reset_evaluate_tracker(ctx)+ ctx.latest_evaluate_result_composition_steer = loaded_results+ data.pop("actionable_targets", None)+ data["composition_targets"] = loaded_result_composition_target_summary(loaded_results)+ data["next_action"] = "compose_extraction"+ data["next_action_reason"] = _EVALUATE_RESULT_COMPOSITION_INSTRUCTION+ # The steer is structured enough to keep under cap without preserving raw page payload.+ _fit_evaluate_steer_under_cap(result, data, keep_raw_page_payload=False)+ LOG.info(+ "copilot_evaluate_result_composition_steer",+ url=url,+ result_container_count=loaded_results.result_container_count,+ table_result_container_count=loaded_results.table_result_container_count,+ )+ # The result is patched in-place; returning False keeps the normal tool-loop guard active.return False+ ctx.latest_evaluate_result_composition_steer = Noneidentities = _actionable_target_identities(parsed)if not identities:- _reset_evaluate_actionable_tracker(ctx)+ _reset_evaluate_tracker(ctx)return Falsesignature = _actionable_target_signature(identities)# Strict full-URL match (fragment included): on an SPA a hash-route change@@ -788,7 +826,7 @@ async def _maybe_steer_evaluate_to_action(if is_repeat:data["next_action"] = "click"data["next_action_reason"] = _EVALUATE_ACTIONABLE_ACT_INSTRUCTION- _fit_evaluate_steer_under_cap(result, data, is_repeat=is_repeat)+ _fit_evaluate_steer_under_cap(result, data, keep_raw_page_payload=not is_repeat)LOG.info("copilot_evaluate_actionable_target_steer",url=url,@@ -798,9 +836,10 @@ async def _maybe_steer_evaluate_to_action()except Exception:data.pop("actionable_targets", None)+ data.pop("composition_targets", None)data.pop("next_action", None)data.pop("next_action_reason", None)- _reset_evaluate_actionable_tracker(ctx)+ _reset_evaluate_tracker(ctx)LOG.warning("copilot_evaluate_actionable_target_steer_failed", exc_info=True)return Falsediff --git a/tests/unit/test_copilot_evaluate_page_evidence.py b/tests/unit/test_copilot_evaluate_page_evidence.pyindex 4f095120a..ea1dfde87 100644--- a/tests/unit/test_copilot_evaluate_page_evidence.py+++ b/tests/unit/test_copilot_evaluate_page_evidence.py@@ -7,6 +7,7 @@ import pytestfrom skyvern.forge.sdk.copilot.build_phase import BuildPhasefrom skyvern.forge.sdk.copilot.context import CopilotContextfrom skyvern.forge.sdk.copilot.request_policy import RequestPolicy+from skyvern.forge.sdk.copilot.result_evidence import LoadedResultCompositionEvidencefrom skyvern.forge.sdk.copilot.tools import (_evaluate_post_hook,_inspect_page_for_composition_impl,@@ -33,6 +34,33 @@ def _ctx() -> CopilotContext:)+ "result",+ [+ {"ok": False, "error": "evaluate failed"},+ {"ok": True},+ {"ok": True, "data": {}},+ {"ok": True, "data": []},+ ],+)+async def test_evaluate_post_hook_resets_steer_on_unusable_result(result: dict[str, object]) -> None:+ ctx = _ctx()+ ctx.last_evaluate_actionable_signature = "stale-signature"+ ctx.last_evaluate_actionable_url = "https://example.test/old"+ ctx.latest_evaluate_result_composition_steer = LoadedResultCompositionEvidence(+ result_container_count=1,+ table_result_container_count=1,+ )++ updated = await _evaluate_post_hook(result, raw={}, ctx=ctx)++ assert updated is result+ assert ctx.last_evaluate_actionable_signature is None+ assert ctx.last_evaluate_actionable_url is None+ assert ctx.latest_evaluate_result_composition_steer is None++@pytest.mark.asyncioasync def test_evaluate_nested_rows_records_interaction_observation_step() -> None:ctx = _ctx()diff --git a/tests/unit/test_copilot_hooks.py b/tests/unit/test_copilot_hooks.pyindex 4675ce706..d808bdc9b 100644--- a/tests/unit/test_copilot_hooks.py+++ b/tests/unit/test_copilot_hooks.py@@ -341,6 +341,27 @@ class TestSchemaOverlay:class TestMCPFailedStepLoopDetection:+ def test_post_hook_context_snapshot_restores_loaded_result_composition_steer(self) -> None:+ from skyvern.forge.sdk.copilot.mcp_adapter import _restore_post_hook_context, _snapshot_post_hook_context+ from skyvern.forge.sdk.copilot.result_evidence import LoadedResultCompositionEvidence++ initial_steer = LoadedResultCompositionEvidence(+ result_container_count=1,+ table_result_container_count=1,+ )+ ctx = SimpleNamespace(+ latest_evaluate_result_composition_steer=initial_steer,+ )++ snapshot = _snapshot_post_hook_context(ctx)+ ctx.latest_evaluate_result_composition_steer = LoadedResultCompositionEvidence(+ result_container_count=2,+ table_result_container_count=0,+ )+ _restore_post_hook_context(ctx, snapshot)++ assert ctx.latest_evaluate_result_composition_steer == initial_steer+@pytest.mark.asyncioasync def test_post_hook_exception_preserves_successful_tool_result(self, monkeypatch: pytest.MonkeyPatch) -> None:from skyvern.forge.sdk.copilot import mcp_adapterdiff --git a/tests/unit/test_copilot_loop_detection_blocker.py b/tests/unit/test_copilot_loop_detection_blocker.pyindex ab4210067..7ad15ac09 100644--- a/tests/unit/test_copilot_loop_detection_blocker.py+++ b/tests/unit/test_copilot_loop_detection_blocker.py@@ -17,6 +17,7 @@ from skyvern.forge.sdk.copilot.blocker_signal import (from skyvern.forge.sdk.copilot.context import CopilotContextfrom skyvern.forge.sdk.copilot.mcp_adapter import SchemaOverlay, SkyvernOverlayMCPServer, _stash_and_emit_loop_blockerfrom skyvern.forge.sdk.copilot.output_policy import CopilotOutputKind, evaluate_output_policy+from skyvern.forge.sdk.copilot.result_evidence import LoadedResultCompositionEvidencefrom skyvern.forge.sdk.copilot.run_outcome import TERMINAL_CHALLENGE_BLOCKER_REASON_CODE, RecordedRunOutcomefrom skyvern.forge.sdk.copilot.tools import _build_loop_blocker_signal, _tool_loop_errorfrom skyvern.forge.sdk.copilot.tools.mcp_hooks import get_skyvern_mcp_alias_map@@ -137,6 +138,60 @@ def test_native_dispatch_consecutive_tool_loop_sets_signal() -> None:assert ctx.turn_halt.kind == TurnHaltKind.LOOP_DETECTED+def test_consecutive_evaluate_loop_with_loaded_results_uses_goal_aware_copy() -> None:+ ctx = _ctx(consecutive_tool_tracker=["evaluate", "evaluate"])+ ctx.latest_evaluate_result_composition_steer = LoadedResultCompositionEvidence(+ result_container_count=1,+ table_result_container_count=1,+ )+ payload = _tool_loop_error(ctx, "evaluate", None)++ assert payload is not None+ assert payload.startswith("LOOP DETECTED: 'evaluate' has been called 3 times consecutively.")+ assert isinstance(ctx.blocker_signal, CopilotToolBlockerSignal)+ assert ctx.blocker_signal.internal_reason_code == "loop_detected_consecutive_same_tool"+ assert "loaded results" in ctx.blocker_signal.user_facing_reason+ assert "extracting the requested information" in ctx.blocker_signal.user_facing_reason+ assert "retrying the same step" not in ctx.blocker_signal.user_facing_reason+ assert dict(ctx.blocker_signal.extra) == {"loop_evidence_tiers": ["loaded_results"]}+ assert_clean_user_facing_text(ctx.blocker_signal.user_facing_reason, blocked_tool="evaluate")+++def test_consecutive_non_evaluate_loop_ignores_loaded_result_steer() -> None:+ ctx = _ctx(consecutive_tool_tracker=["list_credentials", "list_credentials"])+ ctx.latest_evaluate_result_composition_steer = LoadedResultCompositionEvidence(+ result_container_count=1,+ table_result_container_count=1,+ )+ payload = _tool_loop_error(ctx, "list_credentials", None)++ assert payload is not None+ assert isinstance(ctx.blocker_signal, CopilotToolBlockerSignal)+ assert ctx.blocker_signal.internal_reason_code == "loop_detected_consecutive_same_tool"+ assert "loaded results" not in ctx.blocker_signal.user_facing_reason+ assert ctx.blocker_signal.user_facing_reason == (+ "I'm stuck retrying the same step. Tell me what to change and I'll try a different approach."+ )+ assert dict(ctx.blocker_signal.extra) == {}+++def test_consecutive_evaluate_loop_ignores_stale_composition_page_results_without_latest_steer() -> None:+ ctx = _ctx(consecutive_tool_tracker=["evaluate", "evaluate"])+ ctx.composition_page_evidence = {+ "result_containers": [{"tag": "table", "selector": "#old-results", "row_count": 3}],+ }+ payload = _tool_loop_error(ctx, "evaluate", None)++ assert payload is not None+ assert isinstance(ctx.blocker_signal, CopilotToolBlockerSignal)+ assert ctx.blocker_signal.internal_reason_code == "loop_detected_consecutive_same_tool"+ assert ctx.blocker_signal.user_facing_reason == (+ "I'm stuck retrying the same step. Tell me what to change and I'll try a different approach."+ )+ assert dict(ctx.blocker_signal.extra) == {}+ assert_clean_user_facing_text(ctx.blocker_signal.user_facing_reason, blocked_tool="evaluate")++def test_native_and_mcp_paths_produce_equivalent_loop_signal() -> None:"""Native dispatch and the MCP adapter must produce the same signal shapefor the same loop-detection message so a regression in one path can'tdiff --git a/tests/unit/test_copilot_result_evidence.py b/tests/unit/test_copilot_result_evidence.pynew file mode 100644index 000000000..911f9e938--- /dev/null+++ b/tests/unit/test_copilot_result_evidence.py@@ -0,0 +1,77 @@+from __future__ import annotations++from skyvern.forge.sdk.copilot.result_evidence import loaded_result_composition_evidence_from_page+++def test_loaded_result_evidence_counts_populated_tables() -> None:+ evidence = loaded_result_composition_evidence_from_page(+ {+ "result_containers": [+ {"tag": "table", "selector": "#results", "row_count": 2},+ {"selector": "#cards", "sample_rows": ["May 2026 statement available"]},+ ],+ }+ )++ assert evidence is not None+ assert evidence.result_container_count == 2+ assert evidence.table_result_container_count == 1+++def test_caption_only_result_container_is_not_composition_evidence() -> None:+ evidence = loaded_result_composition_evidence_from_page(+ {+ "result_containers": [+ {"selector": "#count", "text": "12 matching records"},+ {"selector": "#results", "text": "3 matching records"},+ {"selector": "#none", "text": "No results found"},+ {"selector": "#empty", "text": "0 results found"},+ ],+ }+ )++ assert evidence is None+++def test_result_container_text_uses_non_ui_chrome_tokens() -> None:+ evidence = loaded_result_composition_evidence_from_page(+ {"result_containers": [{"selector": "#results", "text": "Statement May 2026 available"}]}+ )++ assert evidence is not None+ assert evidence.result_container_count == 1+++def test_standalone_no_can_be_result_payload_text() -> None:+ evidence = loaded_result_composition_evidence_from_page(+ {"result_containers": [{"selector": "#results", "text": "No"}]}+ )++ assert evidence is not None+ assert evidence.result_container_count == 1+++def test_transient_status_result_container_is_not_composition_evidence() -> None:+ evidence = loaded_result_composition_evidence_from_page(+ {+ "result_containers": [+ {"selector": "#loading", "text": "Loading..."},+ {"selector": "#wait", "text": "Please wait"},+ {"selector": "#pagination", "text": "Showing 1-20 of 100"},+ ],+ }+ )++ assert evidence is None+++def test_deep_result_container_payload_does_not_recurse_unbounded() -> None:+ nested: object = "value"+ for _ in range(20):+ nested = [nested]++ evidence = loaded_result_composition_evidence_from_page(+ {"result_containers": [{"selector": "#results", "sample_rows": [nested]}]}+ )++ assert evidence is Nonediff --git a/tests/unit/test_evaluate_actionable_target.py b/tests/unit/test_evaluate_actionable_target.pyindex 447ea47d3..a36ea0392 100644--- a/tests/unit/test_evaluate_actionable_target.py+++ b/tests/unit/test_evaluate_actionable_target.py@@ -33,6 +33,36 @@ def _full_packet() -> dict:}+def _result_table_packet() -> dict:+ return {+ "forms": [],+ "navigation_targets": [{"selector": "#details", "text": "Details", "href": "/details"}],+ "result_containers": [+ {+ "tag": "table",+ "selector": "#results",+ "text": "Results",+ "row_count": 2,+ "sample_rows": ["Result A May 2026 $42.00", "Result B May 2026 $51.00"],+ }+ ],+ }+++def _row_result_packet() -> dict:+ return {+ "forms": [],+ "navigation_targets": [],+ "result_containers": [+ {+ "selector": "#results",+ "row_count": 1,+ "sample_rows": ["May 2026 statement available"],+ }+ ],+ }++def _ctx() -> AgentContext:return AgentContext.__new__(AgentContext)@@ -76,6 +106,60 @@ async def test_first_evaluate_names_targets_without_next_action(monkeypatch) ->assert "next_action" not in result["data"]+async def test_first_loaded_result_table_steers_to_composition_not_click_even_with_actionable_targets(+ monkeypatch,+) -> None:+ async def fake_evidence(ctx, *, url):+ return _result_table_packet()++ monkeypatch.setattr(scouting, "_scout_act_observe_page_evidence", fake_evidence)+ ctx = _ctx()+ ctx.last_evaluate_actionable_signature = "stale"+ ctx.last_evaluate_actionable_url = "https://example.com/previous"+ result = {+ "ok": True,+ "data": {+ "url": "https://example.com/results",+ "actionable_targets": [{"selector": "#details", "text": "Details"}],+ },+ }++ await scouting._maybe_steer_evaluate_to_action(ctx, result, url="https://example.com/results")++ assert result["data"].get("next_action") == "compose_extraction"+ assert "extract" in result["data"]["next_action_reason"]+ assert "click" not in result["data"]["next_action_reason"].lower()+ assert "actionable_targets" not in result["data"]+ assert result["data"]["composition_targets"] == {+ "result_container_count": 1,+ "table_result_container_count": 1,+ }+ assert ctx.latest_evaluate_result_composition_steer.result_container_count == 1+ assert ctx.last_evaluate_actionable_signature is None+ assert ctx.last_evaluate_actionable_url is None+++async def test_first_loaded_result_rows_without_table_marker_steers_to_composition(monkeypatch) -> None:+ async def fake_evidence(ctx, *, url):+ return _row_result_packet()++ monkeypatch.setattr(scouting, "_scout_act_observe_page_evidence", fake_evidence)+ ctx = _ctx()+ ctx.last_evaluate_actionable_signature = None+ ctx.last_evaluate_actionable_url = None+ result = {"ok": True, "data": {"url": "https://example.com/results"}}++ await scouting._maybe_steer_evaluate_to_action(ctx, result, url="https://example.com/results")++ assert result["data"].get("next_action") == "compose_extraction"+ assert result["data"]["composition_targets"] == {+ "result_container_count": 1,+ "table_result_container_count": 0,+ }++@pytest.mark.asyncioasync def test_repeat_evaluate_same_page_steers_to_click(monkeypatch) -> None:async def fake_evidence(ctx, *, url):@@ -162,6 +246,28 @@ async def test_repeat_scalar_result_sheds_under_cap(monkeypatch) -> None:assert '"next_action"' in serialized[: scouting._RECENT_TOOL_OUTPUT_CHAR_CAP]+async def test_first_loaded_result_table_sheds_large_payload_but_keeps_composition_steer(monkeypatch) -> None:+ ctx = _ctx()+ ctx.last_evaluate_actionable_signature = None+ ctx.last_evaluate_actionable_url = None+ url = "https://app.example.com/results"++ async def fake(c, *, url):+ return _result_table_packet()++ monkeypatch.setattr(scouting, "_scout_act_observe_page_evidence", fake)+ big = "x" * 10000+ result = {"ok": True, "data": {"url": url, "result": {"html": big, "text": big}}}+ await scouting._maybe_steer_evaluate_to_action(ctx, result, url=url)+ serialized = json.dumps(result, default=str)+ assert result["data"].get("next_action") == "compose_extraction"+ assert result["data"].get("composition_targets")+ assert len(serialized) <= scouting._RECENT_TOOL_OUTPUT_CHAR_CAP+ assert '"next_action"' in serialized[: scouting._RECENT_TOOL_OUTPUT_CHAR_CAP]+ assert "compose_extraction" in serialized++@pytest.mark.asyncioasync def test_repeat_nested_result_sheds_under_cap(monkeypatch) -> None:ctx = _ctx()@@ -256,8 +362,9 @@ async def test_probe_none_leaves_data_untouched_and_resets(monkeypatch) -> None:@pytest.mark.asyncioasync def test_probe_empty_collections_no_steer(monkeypatch) -> None:ctx = _ctx()- ctx.last_evaluate_actionable_signature = None+ ctx.last_evaluate_actionable_signature = "seed"ctx.last_evaluate_actionable_url = None+ ctx.latest_evaluate_result_composition_steer = object()async def fake(c, *, url):return {"forms": [], "navigation_targets": [], "result_containers": []}@@ -266,6 +373,7 @@ async def test_probe_empty_collections_no_steer(monkeypatch) -> None:result = {"ok": True, "data": {"url": "https://app.example.com/x"}}await scouting._maybe_steer_evaluate_to_action(ctx, result, url="https://app.example.com/x")assert "actionable_targets" not in result["data"]+ assert ctx.latest_evaluate_result_composition_steer is None@pytest.mark.asyncio