Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
SKY-13517 Stop ending the copilot's turn because a page looked blocked (#8060)
Andrew Neilson6ca0f9d9e9d1
diff --git a/skyvern/forge/sdk/copilot/code_block_synthesis.py b/skyvern/forge/sdk/copilot/code_block_synthesis.pyindex 5848a2a8c..0398cb022 100644--- a/skyvern/forge/sdk/copilot/code_block_synthesis.py+++ b/skyvern/forge/sdk/copilot/code_block_synthesis.py@@ -103,10 +103,11 @@ CREDENTIAL_FILL_CODE_PATTERN = re.compile(r"\.fill\(\s*(?:[A-Za-z_]\w*\.\w+|awai# Credential fields the scout must fill live before a code block reading them may persist;# `.otp()` resolves at runtime only, so totp never requires (or credits) a live scout fill.LIVE_SCOUT_CREDENTIAL_FIELDS = frozenset({"username", "password"})+ONE_TIME_CODE_CREDENTIAL_FIELD = "totp"def credential_fill_source(locator_expr: str, param_key: str, field: str) -> str:- if field == "totp":+ if field == ONE_TIME_CODE_CREDENTIAL_FIELD:return f"await {locator_expr}.fill(await {param_key}.otp())"return f"await {locator_expr}.fill({param_key}.{field})"diff --git a/skyvern/forge/sdk/copilot/enforcement.py b/skyvern/forge/sdk/copilot/enforcement.pyindex 147c3fe1a..e51c9136b 100644--- a/skyvern/forge/sdk/copilot/enforcement.py+++ b/skyvern/forge/sdk/copilot/enforcement.py@@ -39,6 +39,7 @@ from skyvern.forge.sdk.copilot.challenge_evidence import composition_challenge_cfrom skyvern.forge.sdk.copilot.code_block_synthesis import (CREDENTIAL_FILL_TOOL_NAME,LIVE_SCOUT_CREDENTIAL_FIELDS,+ ONE_TIME_CODE_CREDENTIAL_FIELD,ObligationFinding,credential_scout_gap,credential_submit_boundary_index,@@ -101,6 +102,7 @@ from skyvern.forge.sdk.copilot.context import (parsed_ask_refs,)from skyvern.forge.sdk.copilot.credential_pause import credential_pause_would_fire, maybe_credential_pause+from skyvern.forge.sdk.copilot.credential_resolution import url_partsfrom skyvern.forge.sdk.copilot.diagnosis_repair_contract import (RepairLoopState,RepairNextAction,@@ -505,12 +507,18 @@ def terminal_challenge_blocker_signal_from_page_evidence(evidence_source: str = "page_evidence",evidence: dict[str, Any] | None = None,) -> CopilotToolBlockerSignal | None:- page_reason = _structured_page_challenge_reason(ctx, evidence)+ packet = evidence if evidence is not None else getattr(ctx, "composition_page_evidence", None)+ page_reason = _structured_page_challenge_reason(ctx, packet)if page_reason is None:return None- carrier = composition_challenge_carrier(- evidence if evidence is not None else getattr(ctx, "composition_page_evidence", None)- )+ if isinstance(packet, Mapping) and one_time_code_fill_supersedes_challenge(ctx, packet):+ LOG.info(+ "copilot_terminal_challenge_declined_credential_served",+ blocked_tool=blocked_tool,+ evidence_source=evidence_source,+ )+ return None+ carrier = composition_challenge_carrier(packet)return _terminal_challenge_halt_signal(ctx,evidence_source=evidence_source,@@ -540,11 +548,107 @@ def _current_page_evidence_candidates(ctx: Any) -> list[dict[str, Any]]:if isinstance(packet, dict):candidates.append(packet)single = getattr(ctx, "composition_page_evidence", None)- if isinstance(single, dict):+ # Both writers alias the packet they appended, so identity keeps one candidate out of two.+ if isinstance(single, dict) and not any(single is packet for packet in candidates):candidates.append(single)return candidates+# Challenge kinds a saved one-time code cannot answer, whoever else is on the page. `unknown` is the+# DOM detector's verdict for every anti-bot vendor it has no name for, so it belongs here too.+_CODE_UNSATISFIABLE_CHALLENGE_KIND_TERMS = (+ "captcha",+ "robot",+ "turnstile",+ "cloudflare",+ "access",+ "human",+ "unknown",+)+++def _observed_page_key(url: Any) -> str | None:+ if not isinstance(url, str) or not url.strip().lower().startswith(("http://", "https://")):+ return None+ parts = url_parts(url.strip())+ return parts[1] if parts else None+++def _one_time_code_fill_targets(ctx: Any) -> set[tuple[str, str]]:+ """(page key, selector) for every saved one-time code this turn filled.++ Keyed by page as well as selector because an observation packet records only the selector, and+ the same selector text recurs across sites.+ """+ targets: set[tuple[str, str]] = set()+ for item in getattr(ctx, "scout_trajectory", None) or []:+ if not isinstance(item, Mapping):+ continue+ if str(item.get("tool_name") or "").strip() != CREDENTIAL_FILL_TOOL_NAME:+ continue+ if str(item.get("credential_field") or "").strip() != ONE_TIME_CODE_CREDENTIAL_FIELD:+ continue+ page = _observed_page_key(item.get("source_url"))+ selector = str(item.get("selector") or "").strip()+ if page and selector:+ targets.add((page, selector))+ return targets+++def _challenge_a_code_cannot_answer(evidence: Mapping[str, Any]) -> bool:+ """A deny-list on purpose: an unrecognized kind stays answerable rather than halting.++ `challenge_state.kind` is free-form vision output, so an allow-list would fail closed on the+ misread this ticket exists to fix — the witnessed failure was labelled `other`.+ """+ controls = evidence.get("challenge_controls")+ if isinstance(controls, list) and interactive_challenge_controls(controls):+ return True+ challenge_state = evidence.get("challenge_state")+ kind = str(challenge_state.get("kind") or "").lower() if isinstance(challenge_state, Mapping) else ""+ return any(term in kind for term in _CODE_UNSATISFIABLE_CHALLENGE_KIND_TERMS)+++def one_time_code_fill_supersedes_challenge(ctx: Any, evidence: Mapping[str, Any]) -> bool:+ """Whether this turn filled a saved one-time code into the observed page after this challenge+ was captured.++ Scoped to challenges older than the code so it can never outlive one: a submit reaches the page+ by routes that mint no observation of their own (an Enter keypress, a block run), so anything+ observed after the fill is left to halt.+ """+ # Nothing is superseded by a packet that reports no challenge, and a caller may hold one whose+ # stop was decided by the run rather than by this page.+ if not isinstance(evidence, dict) or _structured_page_challenge_reason(ctx, evidence) is None:+ return False+ page = _observed_page_key(evidence.get("current_url") or evidence.get("inspected_url"))+ if page is None or _challenge_a_code_cannot_answer(evidence):+ return False+ targets = _one_time_code_fill_targets(ctx)+ if not targets:+ return False+ challenge_seen = False+ for entry in getattr(ctx, "flow_evidence", None) or []:+ if not isinstance(entry, dict):+ continue+ packet = entry.get("evidence")+ if not isinstance(packet, dict):+ continue+ # Both writers alias the appended packet into `composition_page_evidence`; a shallow copy+ # there would silently make this never match, which halts rather than misfires.+ if packet is evidence:+ challenge_seen = True+ continue+ if not challenge_seen or str(packet.get("interaction_tool") or "").strip() != CREDENTIAL_FILL_TOOL_NAME:+ continue+ selector = str(packet.get("interaction_selector") or "").strip()+ # An interaction packet carries the post-interaction URL, so it is attributable to the page+ # it acted on only by the source it recorded.+ if (page, selector) in targets and _observed_page_key(packet.get("interaction_source_url")) == page:+ return True+ return False++def terminal_challenge_blocker_signal_from_current_page_evidence(ctx: Any,*,@@ -569,6 +673,49 @@ def terminal_challenge_blocker_signal_from_current_page_evidence(return None+CURRENT_PAGE_CHALLENGE_ADVISORY_REASON_CODE = "tool_error_current_page_challenge_advisory"+++def current_page_challenge_advisory_signal(+ ctx: Any, *, blocked_tool: str, evidence_source: str = "page_evidence"+) -> CopilotToolBlockerSignal | None:+ """A fire-once advisory in place of the current-page pre-veto: the model sees the challenge+ verdict and decides, holding facts this plane cannot see — what it filled and what it holds.++ Terminal stopping stays with the producers that key on outcomes: a run that failed with+ anti-bot evidence, the loop plane, budget. A page that merely looks like a wall has not+ stopped anything yet.+ """+ signal = terminal_challenge_blocker_signal_from_current_page_evidence(+ ctx, blocked_tool=blocked_tool, evidence_source=evidence_source+ )+ if signal is None:+ return None+ reason = str(signal.extra.get("evidence_reason") or "a verification challenge")+ fired = getattr(ctx, "challenge_advisory_fired_reasons", None)+ if not isinstance(fired, set) or reason in fired:+ return None+ fired.add(reason)+ return CopilotToolBlockerSignal(+ blocker_kind="tool_error",+ agent_steering_text=(+ f"Structured page evidence flags the current page as a verification challenge: {reason}. "+ "This is advisory, not a wall. If the page is asking for something this turn already holds — "+ "for example a one-time code from the resolved credential — fill it, submit it, and continue. "+ "If it is a genuine anti-bot wall you cannot satisfy, do NOT retry the same path; reply to the "+ "user naming the blocker."+ ),+ user_facing_reason="The page may be showing a verification step; I'm checking whether I can complete it.",+ recovery_hint="retry_with_different_tool",+ cleared_by_tools=frozenset({blocked_tool}),+ preserves_workflow_draft=True,+ renders_final_reply=False,+ internal_reason_code=CURRENT_PAGE_CHALLENGE_ADVISORY_REASON_CODE,+ blocked_tool=blocked_tool,+ extra={key: value for key, value in signal.extra.items() if key != "run_outcome_reason_code"},+ )++def _maybe_stash_terminal_challenge_halt(ctx: Any) -> None:if getattr(ctx, "turn_halt", None) is not None:returndiff --git a/skyvern/forge/sdk/copilot/mcp_adapter.py b/skyvern/forge/sdk/copilot/mcp_adapter.pyindex 635a20837..cf13a1f2f 100644--- a/skyvern/forge/sdk/copilot/mcp_adapter.py+++ b/skyvern/forge/sdk/copilot/mcp_adapter.py@@ -36,10 +36,10 @@ from skyvern.forge.sdk.copilot.blocker_signal import ()from skyvern.forge.sdk.copilot.build_phase import _phase_blocker_signalfrom skyvern.forge.sdk.copilot.enforcement import (+ current_page_challenge_advisory_signal,register_no_progress_interaction_click,requested_output_paths_for_derivation,synthesized_block_persistence_signal,- terminal_challenge_blocker_signal_from_current_page_evidence,)from skyvern.forge.sdk.copilot.loop_detection import (detect_failed_tool_step_loop_for_ctx,@@ -144,7 +144,7 @@ class SchemaOverlay:LOG = structlog.get_logger()_INTERNAL_TOOL_ARG_KEYS = frozenset({"_summarized"})-_CURRENT_PAGE_TERMINAL_CHALLENGE_MCP_TOOLS = frozenset(+_CURRENT_PAGE_CHALLENGE_ADVISORY_MCP_TOOLS = frozenset({"click","evaluate",@@ -165,17 +165,11 @@ def _stash_and_emit_loop_blocker(ctx: Any, loop_message: str, tool_name: str) ->return payload-def _stash_and_emit_current_page_terminal_challenge_blocker(ctx: Any, tool_name: str) -> str | None:- signal = terminal_challenge_blocker_signal_from_current_page_evidence(- ctx,- blocked_tool=tool_name,- evidence_source="mcp_page_evidence",- )+def _emit_current_page_challenge_advisory(ctx: Any, tool_name: str) -> str | None:+ signal = current_page_challenge_advisory_signal(ctx, blocked_tool=tool_name, evidence_source="mcp_page_evidence")if signal is None:return None- payload = emit_blocker_signal_payload(ctx, signal)- stash_turn_halt_from_blocker_signal(ctx, signal, source="mcp_current_page_terminal_challenge")- return payload+ return emit_blocker_signal_payload(ctx, signal)def _requested_output_path_choices(schema: dict[str, Any], paths: list[str]) -> dict[str, Any]:@@ -501,17 +495,14 @@ class SkyvernOverlayMCPServer(MCPServer):return _copilot_to_call_tool_result({"ok": False, "error": payload})refresh_held_loop_blocker_evidence(copilot_ctx)- if tool_name in _CURRENT_PAGE_TERMINAL_CHALLENGE_MCP_TOOLS:- terminal_challenge_payload = _stash_and_emit_current_page_terminal_challenge_blocker(- copilot_ctx,- tool_name,- )- if terminal_challenge_payload is not None:- LOG.warning(- "Current page terminal challenge detected, skipping MCP browser tool",+ if tool_name in _CURRENT_PAGE_CHALLENGE_ADVISORY_MCP_TOOLS:+ challenge_advisory_payload = _emit_current_page_challenge_advisory(copilot_ctx, tool_name)+ if challenge_advisory_payload is not None:+ LOG.info(+ "Current page challenge advisory issued for MCP browser tool",tool_name=tool_name,)- return _copilot_to_call_tool_result({"ok": False, "error": terminal_challenge_payload})+ return _copilot_to_call_tool_result({"ok": False, "error": challenge_advisory_payload})persistence_signal = synthesized_block_persistence_signal(copilot_ctx, tool_name, arguments)if persistence_signal is not None:diff --git a/skyvern/forge/sdk/copilot/runtime.py b/skyvern/forge/sdk/copilot/runtime.pyindex 81ea926e1..a75e7bfd1 100644--- a/skyvern/forge/sdk/copilot/runtime.py+++ b/skyvern/forge/sdk/copilot/runtime.py@@ -396,6 +396,8 @@ class AgentContext:# while scouting the goal path, each tagged with how that state was reached.# Feeds the per-acted-page composition gate; never persisted into workflow YAML.flow_evidence: list[dict[str, Any]] = field(default_factory=list)+ # Challenge-advisory reasons already surfaced to the model this turn, so the advisory fires once.+ challenge_advisory_fired_reasons: set[str] = field(default_factory=set)pending_browser_interaction_observation: PendingBrowserInteractionObservation | None = None# In-turn side channel from workflow mutation calls: block label -> flow_evidence# observation step used to ground the newly authored page-acting block.diff --git a/skyvern/forge/sdk/copilot/tools/blockers.py b/skyvern/forge/sdk/copilot/tools/blockers.pyindex bd73aef42..9c4ef5c40 100644--- a/skyvern/forge/sdk/copilot/tools/blockers.py+++ b/skyvern/forge/sdk/copilot/tools/blockers.py@@ -26,8 +26,9 @@ from skyvern.forge.sdk.copilot.completion_verification import ()from skyvern.forge.sdk.copilot.composition_evidence import interactive_challenge_controlsfrom skyvern.forge.sdk.copilot.enforcement import (+ current_page_challenge_advisory_signal,+ one_time_code_fill_supersedes_challenge,synthesized_block_persistence_signal,- terminal_challenge_blocker_signal_from_current_page_evidence,)from skyvern.forge.sdk.copilot.failure_tracking import (PER_TOOL_BUDGET_FAILURE_CATEGORY,@@ -64,7 +65,7 @@ from ._shared import (LOG = structlog.get_logger()-_CURRENT_PAGE_TERMINAL_CHALLENGE_TOOLS = (+_CURRENT_PAGE_CHALLENGE_ADVISORY_TOOLS = (BLOCK_RUNNING_TOOLS| PAGE_INSPECTION_TOOLS| frozenset({"click", "navigate_browser", "press_key", "scroll", "select_option", "type_text"})@@ -667,6 +668,8 @@ def _post_budget_terminal_challenge_signal(reason = _post_run_terminal_challenge_reason(evidence)if not reason:return None+ if one_time_code_fill_supersedes_challenge(ctx, evidence):+ return None# Same-packet adjudication: when the observed page also carries result# evidence, the answer-from-observed-page path outranks a terminal claim.@@ -725,14 +728,10 @@ def _post_budget_terminal_challenge_signal(return _terminal_challenge_signal(reason=reason, arguments=arguments, tool_name=tool_name)-def _current_page_terminal_challenge_signal(+def _current_page_challenge_advisory(ctx: AgentContext, arguments: dict[str, Any] | None, tool_name: str) -> CopilotToolBlockerSignal | None:- signal = terminal_challenge_blocker_signal_from_current_page_evidence(- ctx,- blocked_tool=tool_name,- evidence_source="page_evidence",- )+ signal = current_page_challenge_advisory_signal(ctx, blocked_tool=tool_name)if signal is None:return Nonerequested_labels = _requested_block_label_set(arguments)@@ -1369,6 +1368,9 @@ def _challenge_gated_anti_bot_rerun_signal() -> CopilotToolBlockerSignal | None:if not _last_run_has_terminal_anti_bot_blocker(ctx):return None+ evidence = getattr(ctx, "composition_page_evidence", None)+ if isinstance(evidence, dict) and one_time_code_fill_supersedes_challenge(ctx, evidence):+ return Noneif tool_name == "update_and_run_blocks" and _challenge_gated_proxy_retry_allowed(ctx, arguments):return None@@ -1411,6 +1413,7 @@ LOOP_PLANE_REFUSAL_REASON_CODES: frozenset[str] = frozenset("code_authoring_guardrail_churn","credential_priority_authoring_churn","tool_error_terminal_challenge_blocker",+ "tool_error_current_page_challenge_advisory","tool_error_synthesized_block_persistence_required","tool_error_challenge_gated_submit_disabled","tool_error_non_retriable_nav",@@ -1440,10 +1443,10 @@ def _emit_loop_plane_refusal(ctx: AgentContext, signal: CopilotToolBlockerSignaldef _tool_loop_error(ctx: AgentContext, tool_name: str, arguments: dict[str, Any] | None = None) -> str | None:refresh_held_loop_blocker_evidence(ctx)- if tool_name in _CURRENT_PAGE_TERMINAL_CHALLENGE_TOOLS:- current_page_challenge_signal = _current_page_terminal_challenge_signal(ctx, arguments, tool_name)- if current_page_challenge_signal is not None:- return _emit_loop_plane_refusal(ctx, current_page_challenge_signal)+ if tool_name in _CURRENT_PAGE_CHALLENGE_ADVISORY_TOOLS:+ challenge_advisory = _current_page_challenge_advisory(ctx, arguments, tool_name)+ if challenge_advisory is not None:+ return _emit_loop_plane_refusal(ctx, challenge_advisory)# The diagnosis contract's STOP only runs BETWEEN agent turns, so without this entrypoint# check the model can re-author around an unreachable sandbox before the nudge ever fires.diff --git a/tests/unit/test_copilot_loop_detection_blocker.py b/tests/unit/test_copilot_loop_detection_blocker.pyindex 458e985ba..22d56c1f4 100644--- a/tests/unit/test_copilot_loop_detection_blocker.py+++ b/tests/unit/test_copilot_loop_detection_blocker.py@@ -15,14 +15,16 @@ from skyvern.forge.sdk.copilot.blocker_signal import ()from skyvern.forge.sdk.copilot.config import BlockAuthoringPolicyfrom skyvern.forge.sdk.copilot.context import CopilotContext-from skyvern.forge.sdk.copilot.enforcement import SYNTHESIZED_BLOCK_PERSISTENCE_REASON_CODE+from skyvern.forge.sdk.copilot.enforcement import (+ CURRENT_PAGE_CHALLENGE_ADVISORY_REASON_CODE,+ SYNTHESIZED_BLOCK_PERSISTENCE_REASON_CODE,+)from skyvern.forge.sdk.copilot.loop_detection import tool_step_identityfrom skyvern.forge.sdk.copilot.mcp_adapter import SchemaOverlay, SkyvernOverlayMCPServer, _stash_and_emit_loop_blockerfrom skyvern.forge.sdk.copilot.output_policy import CopilotOutputKind, evaluate_output_policy-from skyvern.forge.sdk.copilot.run_outcome import TERMINAL_CHALLENGE_BLOCKER_REASON_CODE, RecordedRunOutcome+from skyvern.forge.sdk.copilot.run_outcome import RecordedRunOutcomefrom skyvern.forge.sdk.copilot.tools import _build_loop_blocker_signal, _tool_loop_errorfrom skyvern.forge.sdk.copilot.tools.mcp_hooks import get_skyvern_mcp_alias_map-from skyvern.forge.sdk.copilot.turn_halt import TurnHaltKindfrom skyvern.forge.sdk.copilot.turn_intent import TurnIntent, TurnIntentAuthority, TurnIntentMode_LEAK_TOKENS = ("safe_reason_code", "LOOP DETECTED:")@@ -355,7 +357,7 @@ def test_fixed_tier_copy_is_clean_for_every_loop_prone_tool(blocked_tool: str, rassert_clean_user_facing_text(signal.user_facing_reason, blocked_tool=blocked_tool)-def test_native_tool_loop_error_terminal_challenge_preempts_same_tool_loop() -> None:+def test_native_tool_loop_error_challenge_advisory_fires_once_then_loop_plane_owns() -> None:ctx = _ctx(consecutive_tool_tracker=_streak("evaluate", {"expression": "document.body.innerText"}))ctx.composition_page_evidence = _current_page_challenge_evidence()@@ -364,21 +366,23 @@ def test_native_tool_loop_error_terminal_challenge_preempts_same_tool_loop() ->assert payload is not Nonesignal = ctx.blocker_signalassert isinstance(signal, CopilotToolBlockerSignal)- assert signal.internal_reason_code == TERMINAL_CHALLENGE_BLOCKER_REASON_CODE+ assert signal.internal_reason_code == CURRENT_PAGE_CHALLENGE_ADVISORY_REASON_CODEassert signal.blocked_tool == "evaluate"assert signal.extra["evidence_source"] == "page_evidence"- assert ctx.turn_halt is not None- assert ctx.turn_halt.kind == TurnHaltKind.ACTIVE_TERMINAL_CHALLENGE+ assert signal.renders_final_reply is False+ assert ctx.turn_halt is None++ assert _tool_loop_error(ctx, "evaluate", {"expression": "document.body.innerText"}) is None@pytest.mark.asyncio-async def test_mcp_browser_tool_terminal_challenge_preempts_same_tool_loop() -> None:+async def test_mcp_browser_tool_challenge_advisory_defers_execution_once() -> None:ctx = _ctx(consecutive_tool_tracker=_streak("evaluate", {"expression": "document.body.innerText"}))ctx.composition_page_evidence = _current_page_challenge_evidence()class _UnexpectedClient:async def call_tool(self, name: str, args: dict, raise_on_error: bool = False) -> object:- raise AssertionError("terminal challenge blocker should skip MCP execution")+ raise AssertionError("the challenge advisory should defer MCP execution")server = SkyvernOverlayMCPServer(transport=None,@@ -396,11 +400,35 @@ async def test_mcp_browser_tool_terminal_challenge_preempts_same_tool_loop() ->assert parsed["ok"] is Falsesignal = ctx.blocker_signalassert isinstance(signal, CopilotToolBlockerSignal)- assert signal.internal_reason_code == TERMINAL_CHALLENGE_BLOCKER_REASON_CODE+ assert signal.internal_reason_code == CURRENT_PAGE_CHALLENGE_ADVISORY_REASON_CODEassert signal.blocked_tool == "evaluate"assert signal.extra["evidence_source"] == "mcp_page_evidence"- assert ctx.turn_halt is not None- assert ctx.turn_halt.kind == TurnHaltKind.ACTIVE_TERMINAL_CHALLENGE+ assert ctx.turn_halt is None+++async def test_mcp_browser_tool_executes_after_the_challenge_advisory_was_delivered() -> None:+ ctx = _ctx()+ ctx.composition_page_evidence = _current_page_challenge_evidence()++ calls: list[str] = []++ class _CountingClient:+ async def call_tool(self, name: str, args: dict, raise_on_error: bool = False) -> _OkResult:+ calls.append(name)+ return _OkResult()++ server = _dispatch_server(ctx, _CountingClient())++ first = await server.call_tool("evaluate", {"expression": "document.title"})+ assert first.isError is True+ assert calls == []+ assert ctx.turn_halt is None++ second = await server.call_tool("evaluate", {"expression": "document.title"})+ assert second.isError is False+ assert calls == ["evaluate"]+ assert ctx.turn_halt is None@pytest.mark.asynciodiff --git a/tests/unit/test_copilot_per_tool_budget.py b/tests/unit/test_copilot_per_tool_budget.pyindex d91070253..c18f6a63b 100644--- a/tests/unit/test_copilot_per_tool_budget.py+++ b/tests/unit/test_copilot_per_tool_budget.py@@ -32,6 +32,7 @@ from skyvern.forge.sdk.copilot.diagnosis_repair_contract import (VerificationResult,)from skyvern.forge.sdk.copilot.enforcement import (+ CURRENT_PAGE_CHALLENGE_ADVISORY_REASON_CODE,MAX_PER_TOOL_BUDGET_NUDGES,POST_ANTI_BOT_FAILED_TEST_NUDGE,POST_FAILED_TEST_NUDGE,@@ -44,10 +45,6 @@ from skyvern.forge.sdk.copilot.failure_tracking import (PER_TOOL_BUDGET_FAILURE_CATEGORY,compute_failure_signature,)-from skyvern.forge.sdk.copilot.run_outcome import (- TERMINAL_CHALLENGE_BLOCKER_REASON_CODE,- TERMINAL_CHALLENGE_RUN_OUTCOME_REASON_CODE,-)from skyvern.forge.sdk.copilot.tools import (WatchdogExitReason,_composition_anti_bot_reason,@@ -64,6 +61,7 @@ from skyvern.forge.sdk.copilot.tools import (_tool_loop_error,_watchdog_user_failure_reason,)+from skyvern.forge.sdk.copilot.tools.blockers import _challenge_gated_anti_bot_rerun_signalfrom skyvern.forge.sdk.copilot.turn_halt import CopilotTurnHalt, TurnHaltKindfrom tests.unit.conftest import make_copilot_context as _fresh_context@@ -159,7 +157,7 @@ def test_composition_anti_bot_reason_reads_typed_challenge_state_without_legacy_assert "challenge-gated disabled submit/search control: Search" in reason-def test_block_running_halts_on_current_page_terminal_challenge_before_repair_loop() -> None:+def test_block_running_gets_the_challenge_advisory_before_repair_loop() -> None:ctx = _fresh_context()ctx.composition_page_evidence = {"observed_after_workflow_run": True,@@ -176,12 +174,11 @@ def test_block_running_halts_on_current_page_terminal_challenge_before_repair_loassert msg is not Noneassert ctx.blocker_signal is not None- assert ctx.blocker_signal.internal_reason_code == TERMINAL_CHALLENGE_BLOCKER_REASON_CODE- assert ctx.blocker_signal.extra["run_outcome_reason_code"] == TERMINAL_CHALLENGE_RUN_OUTCOME_REASON_CODE+ assert ctx.blocker_signal.internal_reason_code == CURRENT_PAGE_CHALLENGE_ADVISORY_REASON_CODEassert ctx.blocker_signal.extra["evidence_source"] == "page_evidence"assert ctx.blocker_signal.extra["block_labels"] == ["search_lookup"]- assert ctx.turn_halt is not None- assert ctx.turn_halt.kind == TurnHaltKind.ACTIVE_TERMINAL_CHALLENGE+ assert ctx.turn_halt is None+ assert _tool_loop_error(ctx, "update_and_run_blocks", {"block_labels": ["search_lookup"]}) is Nonedef test_block_running_does_not_halt_current_page_challenge_when_results_are_present() -> None:@@ -202,7 +199,7 @@ def test_block_running_does_not_halt_current_page_challenge_when_results_are_preassert ctx.turn_halt is None-def test_block_running_halts_current_page_challenge_with_empty_result_shell() -> None:+def test_block_running_gets_the_challenge_advisory_on_an_empty_result_shell() -> None:ctx = _fresh_context()ctx.composition_page_evidence = {"observed_after_workflow_run": True,@@ -220,9 +217,8 @@ def test_block_running_halts_current_page_challenge_with_empty_result_shell() ->assert msg is not Noneassert ctx.blocker_signal is not None- assert ctx.blocker_signal.internal_reason_code == TERMINAL_CHALLENGE_BLOCKER_REASON_CODE- assert ctx.turn_halt is not None- assert ctx.turn_halt.kind == TurnHaltKind.ACTIVE_TERMINAL_CHALLENGE+ assert ctx.blocker_signal.internal_reason_code == CURRENT_PAGE_CHALLENGE_ADVISORY_REASON_CODE+ assert ctx.turn_halt is Nonedef test_block_running_after_budget_current_url_requires_inspection_first() -> None:@@ -712,7 +708,7 @@ def test_tool_loop_error_allows_new_smaller_label_after_budget_problem() -> None)-def test_tool_loop_error_blocks_challenge_gated_disabled_submit_rerun() -> None:+def test_challenge_advisory_then_attempt_keyed_rerun_stop_on_unchanged_path() -> None:ctx = _fresh_context()ctx.workflow_yaml = """title: Registry standard credential lookup@@ -749,13 +745,27 @@ workflow_definition:)assert msg is not None- assert "Structured challenge evidence confirms this path is blocked" in msgassert ctx.blocker_signal is not None- assert ctx.blocker_signal.internal_reason_code == TERMINAL_CHALLENGE_BLOCKER_REASON_CODE- assert ctx.blocker_signal.renders_final_reply is True+ assert ctx.blocker_signal.internal_reason_code == CURRENT_PAGE_CHALLENGE_ADVISORY_REASON_CODE+ assert ctx.blocker_signal.renders_final_reply is False+ assert ctx.turn_halt is None+ repeat = _tool_loop_error(+ ctx,+ "update_and_run_blocks",+ {+ "workflow_yaml": ctx.workflow_yaml,+ "block_labels": ["fill_sample_record_search"],+ },+ )-def test_tool_loop_error_blocks_current_page_challenge_with_changed_proxy_location() -> None:+ assert repeat is not None+ assert ctx.latest_tool_blocker_signal is not None+ assert ctx.latest_tool_blocker_signal.internal_reason_code == "tool_error_challenge_gated_submit_disabled"+ assert ctx.latest_tool_blocker_signal.renders_final_reply is True+++def test_challenge_advisory_then_single_proxy_retry_then_attempt_keyed_stop() -> None:ctx = _fresh_context()ctx.last_failed_workflow_yaml = """title: Registry standard credential lookup@@ -798,10 +808,36 @@ workflow_definition:)assert msg is not None- assert "do NOT try a proxy/location switch" in msg- assert ctx.challenge_gated_proxy_retry_count == 0assert ctx.blocker_signal is not None- assert ctx.blocker_signal.internal_reason_code == TERMINAL_CHALLENGE_BLOCKER_REASON_CODE+ assert ctx.blocker_signal.internal_reason_code == CURRENT_PAGE_CHALLENGE_ADVISORY_REASON_CODE+ assert ctx.turn_halt is None++ proxy_retry = _tool_loop_error(+ ctx,+ "update_and_run_blocks",+ {+ "workflow_yaml": changed_proxy_yaml,+ "block_labels": ["fill_sample_record_search"],+ },+ )++ assert proxy_retry is None++ exhausted = _tool_loop_error(+ ctx,+ "update_and_run_blocks",+ {+ "workflow_yaml": changed_proxy_yaml,+ "block_labels": ["fill_sample_record_search"],+ },+ )++ assert exhausted is not None+ assert ctx.latest_tool_blocker_signal is not None+ assert ctx.latest_tool_blocker_signal.internal_reason_code == "tool_error_challenge_gated_submit_disabled"+ assert ctx.challenge_gated_proxy_retry_count == 1+ assert ctx.blocker_signal is not None+ assert ctx.blocker_signal.internal_reason_code == CURRENT_PAGE_CHALLENGE_ADVISORY_REASON_CODEdef test_tool_loop_error_blocks_post_budget_challenge_even_with_changed_proxy_location() -> None:@@ -1635,3 +1671,113 @@ def test_rendered_challenge_widget_still_produces_terminal_reason() -> None:reason = _post_run_terminal_challenge_reason(evidence)assert reason is not Noneassert "disabled" in reason+++_CODE_PAGE_URL = "https://example.com/account/login/totp?next=%2Freports"+++def _code_page_challenge_evidence() -> dict:+ """A challenge on a one-time-code page, of a kind a saved code can answer."""+ return _challenge_gated_post_run_evidence(+ current_url=_CODE_PAGE_URL,+ challenge_state={+ "detected": True,+ "kind": "other",+ "requires_human_verification": True,+ "gates_submit_controls": True,+ "gated_submit_controls": [{"text": "Next", "disabled": True}],+ },+ )+++def _arm_code_already_filled(ctx: CopilotContext, evidence: dict) -> None:+ """Put the challenge in flow evidence, then a one-time-code fill observed after it."""+ ctx.composition_page_evidence = evidence+ ctx.scout_trajectory = [+ {+ "tool_name": "fill_credential_field",+ "selector": "#totp",+ "source_url": _CODE_PAGE_URL,+ "credential_field": "totp",+ }+ ]+ ctx.flow_evidence = [+ {"evidence": evidence},+ {+ "evidence": {+ "current_url": _CODE_PAGE_URL,+ "interaction_tool": "fill_credential_field",+ "interaction_selector": "#totp",+ "interaction_source_url": _CODE_PAGE_URL,+ }+ },+ ]+++def _arm_challenge_gated_rerun(ctx: CopilotContext) -> None:+ ctx.last_test_anti_bot = "challenge-gated disabled submit/search control"+ ctx.last_test_failure_reason = "blocker: the submit control was disabled by a verification challenge"+++def test_post_budget_challenge_declines_a_code_page_the_turn_already_answered() -> None:+ ctx = _armed_post_run_context()+ _arm_code_already_filled(ctx, _code_page_challenge_evidence())++ assert _post_budget_terminal_challenge_signal(ctx, _BLOCK_LABEL_ARGS, "update_and_run_blocks") is None+++def test_post_budget_challenge_still_fires_without_a_one_time_code_fill() -> None:+ ctx = _armed_post_run_context()+ ctx.composition_page_evidence = _code_page_challenge_evidence()++ assert _post_budget_terminal_challenge_signal(ctx, _BLOCK_LABEL_ARGS, "update_and_run_blocks") is not None+++def test_challenge_gated_rerun_declines_a_code_page_the_turn_already_answered() -> None:+ ctx = _armed_post_run_context()+ _arm_challenge_gated_rerun(ctx)+ _arm_code_already_filled(ctx, _code_page_challenge_evidence())++ assert _challenge_gated_anti_bot_rerun_signal(ctx, _BLOCK_LABEL_ARGS, "get_run_results") is None+++def test_challenge_gated_rerun_still_fires_without_a_one_time_code_fill() -> None:+ ctx = _armed_post_run_context()+ _arm_challenge_gated_rerun(ctx)+ ctx.composition_page_evidence = _code_page_challenge_evidence()++ assert _challenge_gated_anti_bot_rerun_signal(ctx, _BLOCK_LABEL_ARGS, "get_run_results") is not None+++def test_challenge_gated_rerun_still_fires_when_no_page_evidence_was_captured() -> None:+ """With no page observation there is nothing to order the fill against, so the stop stands."""+ ctx = _armed_post_run_context()+ _arm_challenge_gated_rerun(ctx)+ ctx.last_test_failure_reason = "blocker: bot detection challenge interrupted the run"+ ctx.composition_page_evidence = None+ ctx.scout_trajectory = [+ {+ "tool_name": "fill_credential_field",+ "selector": "#totp",+ "source_url": _CODE_PAGE_URL,+ "credential_field": "totp",+ }+ ]++ assert _challenge_gated_anti_bot_rerun_signal(ctx, _BLOCK_LABEL_ARGS, "get_run_results") is not None+++def test_challenge_gated_rerun_still_fires_on_a_benign_packet_after_a_code_fill() -> None:+ """A run-reported bot wall is not answered by a code typed into an unchallenged page."""+ ctx = _armed_post_run_context()+ _arm_challenge_gated_rerun(ctx)+ ctx.last_test_failure_reason = "blocker: bot detection challenge interrupted the run"+ benign = {+ "current_url": _CODE_PAGE_URL,+ "inspected_url": _CODE_PAGE_URL,+ "observed_after_workflow_run": True,+ "forms": [],+ }+ _arm_code_already_filled(ctx, benign)++ assert _challenge_gated_anti_bot_rerun_signal(ctx, _BLOCK_LABEL_ARGS, "get_run_results") is not Nonediff --git a/tests/unit/test_copilot_turn_halt.py b/tests/unit/test_copilot_turn_halt.pyindex 9d326bf90..ef871788d 100644--- a/tests/unit/test_copilot_turn_halt.py+++ b/tests/unit/test_copilot_turn_halt.py@@ -195,9 +195,248 @@ def test_page_challenge_signal_records_explicit_terminal_blocker() -> None:assert signal.blocked_tool == "update_and_run_blocks"+_TOTP_URL = "https://portal.example.com/account/login/totp?next=%2Freports"+_LOGIN_URL = "https://portal.example.com/account/login?next=%2Freports"+_OTHER_TOTP_URL = "https://other.example.com/account/login/totp"+++def _vision_challenge_packet() -> dict[str, object]:+ """A vision-stamped verification verdict on the one-time-code page."""+ return {+ "current_url": _TOTP_URL,+ "inspected_url": _TOTP_URL,+ "observed_after_workflow_run": True,+ "challenge_state": {+ "detected": True,+ "kind": "other",+ "requires_human_verification": True,+ "evidence_source": "vision",+ },+ }+++def _captcha_widget_packet() -> dict[str, object]:+ """A rendered challenge widget on the same page: a control the copilot must actually solve."""+ return {+ "current_url": _TOTP_URL,+ "inspected_url": _TOTP_URL,+ "observed_after_workflow_run": True,+ "challenge_controls": [{"kind": "recaptcha", "selector": "iframe[title='reCAPTCHA']", "visible": True}],+ }+++def _captcha_kind_packet() -> dict[str, object]:+ """A vision-confirmed CAPTCHA carrying no parsed DOM control — named by kind alone."""+ packet = _vision_challenge_packet()+ challenge_state = packet["challenge_state"]+ assert isinstance(challenge_state, dict)+ challenge_state["kind"] = "captcha"+ return packet+++def _unknown_kind_packet() -> dict[str, object]:+ """The DOM detector's verdict for an anti-bot vendor it has no name for."""+ packet = _vision_challenge_packet()+ challenge_state = packet["challenge_state"]+ assert isinstance(challenge_state, dict)+ challenge_state["kind"] = "unknown"+ return packet+++def _interaction_packet(+ tool: str, selector: str, *, landed_url: str = _TOTP_URL, acted_url: str = _TOTP_URL+) -> dict[str, object]:+ return {+ "current_url": landed_url,+ "inspected_url": landed_url,+ "source_tool": "scout_interaction",+ "interaction_tool": tool,+ "interaction_selector": selector,+ "interaction_source_url": acted_url,+ }+++def _interaction_challenge_packet(tool: str, selector: str) -> dict[str, object]:+ """A click's own post-action observation, which merges the parsed page evidence."""+ return {**_vision_challenge_packet(), **_interaction_packet(tool, selector)}+++def _credential_login_ctx(+ *,+ code_filled: bool,+ code_submitted: bool = False,+ challenge_reobserved_before_submit: bool = False,+ challenge_reobserved_after_submit: bool = False,+ challenge_packet: dict[str, object] | None = None,+ intervening_click: bool = False,+ submit_navigated: bool = False,+ enter_submitted: bool = False,+ fill_on_another_page: bool = False,+ fill_acted_elsewhere: bool = False,+) -> SimpleNamespace:+ trajectory: list[dict[str, object]] = [+ {+ "tool_name": "fill_credential_field",+ "selector": "#username",+ "source_url": _LOGIN_URL,+ "credential_field": "username",+ },+ {+ "tool_name": "fill_credential_field",+ "selector": "#password",+ "source_url": _LOGIN_URL,+ "credential_field": "password",+ },+ {"tool_name": "click", "selector": 'button[aria-label="Log in"]', "source_url": _LOGIN_URL},+ ]+ flow_evidence: list[dict[str, object]] = [{"evidence": challenge_packet or _vision_challenge_packet()}]+ if code_filled:+ fill_page = _OTHER_TOTP_URL if fill_on_another_page else _TOTP_URL+ trajectory.append(+ {+ "tool_name": "fill_credential_field",+ "selector": "#totp",+ "source_url": fill_page,+ "credential_field": "totp",+ }+ )+ # A packet whose recorded source is elsewhere must not be credited to the challenged page,+ # even though it landed there and its selector matches.+ acted = _OTHER_TOTP_URL if fill_acted_elsewhere else fill_page+ flow_evidence.append({"evidence": _interaction_packet("fill_credential_field", "#totp", acted_url=acted)})+ if challenge_reobserved_before_submit:+ flow_evidence.append({"evidence": _vision_challenge_packet()})+ if intervening_click:+ trajectory.append({"tool_name": "click", "selector": "#cookie-accept", "source_url": _TOTP_URL})+ flow_evidence.append({"evidence": _interaction_challenge_packet("click", "#cookie-accept")})+ if code_submitted and enter_submitted:+ # An Enter keypress commits the code without minting an interaction observation.+ trajectory.append({"tool_name": "press_key", "key": "Enter", "source_url": _TOTP_URL})+ elif code_submitted:+ trajectory.append({"tool_name": "click", "selector": "#totp-submit", "source_url": _TOTP_URL})+ landed = "https://portal.example.com/reports" if submit_navigated else _TOTP_URL+ flow_evidence.append({"evidence": _interaction_packet("click", "#totp-submit", landed_url=landed)})+ if challenge_reobserved_after_submit:+ flow_evidence.append({"evidence": _vision_challenge_packet()})+ return SimpleNamespace(+ flow_evidence=flow_evidence,+ scout_trajectory=trajectory,+ last_failure_category_top=None,+ last_run_blocks_workflow_run_id="wr_000000000000000000",+ )+++def _backstop_ctx(*, code_filled: bool) -> SimpleNamespace:+ login = _credential_login_ctx(code_filled=code_filled)+ return _halt_ctx(+ flow_evidence=login.flow_evidence,+ scout_trajectory=login.scout_trajectory,+ composition_page_evidence=login.flow_evidence[0]["evidence"],+ last_run_outcome=None,+ last_test_ok=False,+ last_test_anti_bot=True,+ )+++ ("code_filled", "expect_halt"),+ [(True, False), (False, True)],+ ids=["backstop_declines_the_stale_packet_the_code_answered", "backstop_still_halts_without_a_code_fill"],+)+def test_finalize_backstop_shares_the_credential_served_decline(code_filled: bool, expect_halt: bool) -> None:+ ctx = _backstop_ctx(code_filled=code_filled)++ _maybe_stash_terminal_challenge_halt(ctx)++ assert (ctx.turn_halt is not None) is expect_halt++@pytest.mark.parametrize(("ctx", "blocked_tool", "expect_signal"),[+ pytest.param(+ _credential_login_ctx(code_filled=True),+ "click",+ False,+ id="declines_on_a_challenge_seen_before_the_code_was_filled",+ ),+ pytest.param(+ _credential_login_ctx(code_filled=True, code_submitted=True),+ "evaluate",+ False,+ id="stale_challenge_stays_declined_for_the_tool_after_the_submit",+ ),+ pytest.param(+ _credential_login_ctx(code_filled=True, challenge_reobserved_before_submit=True),+ "click",+ True,+ id="halts_on_a_challenge_observed_after_the_fill_even_before_a_submit",+ ),+ pytest.param(+ _credential_login_ctx(code_filled=True, code_submitted=True, enter_submitted=True),+ "evaluate",+ False,+ id="an_enter_submit_cannot_extend_the_decline_past_the_fill",+ ),+ pytest.param(+ _credential_login_ctx(code_filled=True, fill_on_another_page=True),+ "click",+ True,+ id="halts_when_the_code_was_filled_into_a_different_page",+ ),+ pytest.param(+ _credential_login_ctx(code_filled=True, fill_acted_elsewhere=True),+ "click",+ True,+ id="halts_when_the_fill_packet_acted_on_a_different_page_than_it_landed_on",+ ),+ pytest.param(+ _credential_login_ctx(code_filled=True, challenge_packet=_unknown_kind_packet()),+ "click",+ True,+ id="halts_on_an_unnamed_anti_bot_kind_the_dom_detector_could_not_classify",+ ),+ pytest.param(+ _credential_login_ctx(code_filled=True, code_submitted=True, challenge_reobserved_after_submit=True),+ "click",+ True,+ id="halts_when_the_page_still_demands_a_code_after_the_submit",+ ),+ pytest.param(+ _credential_login_ctx(code_filled=True, intervening_click=True),+ "click",+ True,+ id="halts_when_an_intervening_click_reobserves_the_challenge_itself",+ ),+ pytest.param(+ _credential_login_ctx(+ code_filled=True,+ code_submitted=True,+ submit_navigated=True,+ challenge_reobserved_after_submit=True,+ ),+ "click",+ True,+ id="halts_after_a_navigating_submit_when_the_page_still_demands_a_code",+ ),+ pytest.param(+ _credential_login_ctx(code_filled=True, challenge_packet=_captcha_widget_packet()),+ "click",+ True,+ id="halts_on_a_rendered_challenge_widget_the_code_cannot_satisfy",+ ),+ pytest.param(+ _credential_login_ctx(code_filled=True, challenge_packet=_captcha_kind_packet()),+ "click",+ True,+ id="halts_on_a_vision_named_captcha_with_no_dom_control",+ ),+ pytest.param(+ _credential_login_ctx(code_filled=False),+ "click",+ True,+ id="halts_when_no_credential_fill_reached_the_challenged_page",+ ),pytest.param(SimpleNamespace(last_failure_category_top=None,