Skip to content

Automate browser based workflows with AI

SKY-13517 Stop ending the copilot's turn because a page looked blocked (#8060)

Andrew Neilson6ca0f9d9e9d1
diff --git a/skyvern/forge/sdk/copilot/code_block_synthesis.py b/skyvern/forge/sdk/copilot/code_block_synthesis.py
index 5848a2a8c..0398cb022 100644
--- a/skyvern/forge/sdk/copilot/code_block_synthesis.py
+++ b/skyvern/forge/sdk/copilot/code_block_synthesis.py
@@ -103,10 +103,11 @@ CREDENTIAL_FILL_CODE_PATTERN = re.compile(r"\.fill\(\s*(?:[A-Za-z_]\w*\.\w+|awai
# Credential fields the scout must fill live before a code block reading them may persist;
# `.otp()` resolves at runtime only, so totp never requires (or credits) a live scout fill.
LIVE_SCOUT_CREDENTIAL_FIELDS = frozenset({"username", "password"})
+ONE_TIME_CODE_CREDENTIAL_FIELD = "totp"
def credential_fill_source(locator_expr: str, param_key: str, field: str) -> str:
- if field == "totp":
+ if field == ONE_TIME_CODE_CREDENTIAL_FIELD:
return f"await {locator_expr}.fill(await {param_key}.otp())"
return f"await {locator_expr}.fill({param_key}.{field})"
diff --git a/skyvern/forge/sdk/copilot/enforcement.py b/skyvern/forge/sdk/copilot/enforcement.py
index 147c3fe1a..e51c9136b 100644
--- a/skyvern/forge/sdk/copilot/enforcement.py
+++ b/skyvern/forge/sdk/copilot/enforcement.py
@@ -39,6 +39,7 @@ from skyvern.forge.sdk.copilot.challenge_evidence import composition_challenge_c
from skyvern.forge.sdk.copilot.code_block_synthesis import (
CREDENTIAL_FILL_TOOL_NAME,
LIVE_SCOUT_CREDENTIAL_FIELDS,
+ ONE_TIME_CODE_CREDENTIAL_FIELD,
ObligationFinding,
credential_scout_gap,
credential_submit_boundary_index,
@@ -101,6 +102,7 @@ from skyvern.forge.sdk.copilot.context import (
parsed_ask_refs,
)
from skyvern.forge.sdk.copilot.credential_pause import credential_pause_would_fire, maybe_credential_pause
+from skyvern.forge.sdk.copilot.credential_resolution import url_parts
from skyvern.forge.sdk.copilot.diagnosis_repair_contract import (
RepairLoopState,
RepairNextAction,
@@ -505,12 +507,18 @@ def terminal_challenge_blocker_signal_from_page_evidence(
evidence_source: str = "page_evidence",
evidence: dict[str, Any] | None = None,
) -> CopilotToolBlockerSignal | None:
- page_reason = _structured_page_challenge_reason(ctx, evidence)
+ packet = evidence if evidence is not None else getattr(ctx, "composition_page_evidence", None)
+ page_reason = _structured_page_challenge_reason(ctx, packet)
if page_reason is None:
return None
- carrier = composition_challenge_carrier(
- evidence if evidence is not None else getattr(ctx, "composition_page_evidence", None)
- )
+ if isinstance(packet, Mapping) and one_time_code_fill_supersedes_challenge(ctx, packet):
+ LOG.info(
+ "copilot_terminal_challenge_declined_credential_served",
+ blocked_tool=blocked_tool,
+ evidence_source=evidence_source,
+ )
+ return None
+ carrier = composition_challenge_carrier(packet)
return _terminal_challenge_halt_signal(
ctx,
evidence_source=evidence_source,
@@ -540,11 +548,107 @@ def _current_page_evidence_candidates(ctx: Any) -> list[dict[str, Any]]:
if isinstance(packet, dict):
candidates.append(packet)
single = getattr(ctx, "composition_page_evidence", None)
- if isinstance(single, dict):
+ # Both writers alias the packet they appended, so identity keeps one candidate out of two.
+ if isinstance(single, dict) and not any(single is packet for packet in candidates):
candidates.append(single)
return candidates
+# Challenge kinds a saved one-time code cannot answer, whoever else is on the page. `unknown` is the
+# DOM detector's verdict for every anti-bot vendor it has no name for, so it belongs here too.
+_CODE_UNSATISFIABLE_CHALLENGE_KIND_TERMS = (
+ "captcha",
+ "robot",
+ "turnstile",
+ "cloudflare",
+ "access",
+ "human",
+ "unknown",
+)
+
+
+def _observed_page_key(url: Any) -> str | None:
+ if not isinstance(url, str) or not url.strip().lower().startswith(("http://", "https://")):
+ return None
+ parts = url_parts(url.strip())
+ return parts[1] if parts else None
+
+
+def _one_time_code_fill_targets(ctx: Any) -> set[tuple[str, str]]:
+ """(page key, selector) for every saved one-time code this turn filled.
+
+ Keyed by page as well as selector because an observation packet records only the selector, and
+ the same selector text recurs across sites.
+ """
+ targets: set[tuple[str, str]] = set()
+ for item in getattr(ctx, "scout_trajectory", None) or []:
+ if not isinstance(item, Mapping):
+ continue
+ if str(item.get("tool_name") or "").strip() != CREDENTIAL_FILL_TOOL_NAME:
+ continue
+ if str(item.get("credential_field") or "").strip() != ONE_TIME_CODE_CREDENTIAL_FIELD:
+ continue
+ page = _observed_page_key(item.get("source_url"))
+ selector = str(item.get("selector") or "").strip()
+ if page and selector:
+ targets.add((page, selector))
+ return targets
+
+
+def _challenge_a_code_cannot_answer(evidence: Mapping[str, Any]) -> bool:
+ """A deny-list on purpose: an unrecognized kind stays answerable rather than halting.
+
+ `challenge_state.kind` is free-form vision output, so an allow-list would fail closed on the
+ misread this ticket exists to fix — the witnessed failure was labelled `other`.
+ """
+ controls = evidence.get("challenge_controls")
+ if isinstance(controls, list) and interactive_challenge_controls(controls):
+ return True
+ challenge_state = evidence.get("challenge_state")
+ kind = str(challenge_state.get("kind") or "").lower() if isinstance(challenge_state, Mapping) else ""
+ return any(term in kind for term in _CODE_UNSATISFIABLE_CHALLENGE_KIND_TERMS)
+
+
+def one_time_code_fill_supersedes_challenge(ctx: Any, evidence: Mapping[str, Any]) -> bool:
+ """Whether this turn filled a saved one-time code into the observed page after this challenge
+ was captured.
+
+ Scoped to challenges older than the code so it can never outlive one: a submit reaches the page
+ by routes that mint no observation of their own (an Enter keypress, a block run), so anything
+ observed after the fill is left to halt.
+ """
+ # Nothing is superseded by a packet that reports no challenge, and a caller may hold one whose
+ # stop was decided by the run rather than by this page.
+ if not isinstance(evidence, dict) or _structured_page_challenge_reason(ctx, evidence) is None:
+ return False
+ page = _observed_page_key(evidence.get("current_url") or evidence.get("inspected_url"))
+ if page is None or _challenge_a_code_cannot_answer(evidence):
+ return False
+ targets = _one_time_code_fill_targets(ctx)
+ if not targets:
+ return False
+ challenge_seen = False
+ for entry in getattr(ctx, "flow_evidence", None) or []:
+ if not isinstance(entry, dict):
+ continue
+ packet = entry.get("evidence")
+ if not isinstance(packet, dict):
+ continue
+ # Both writers alias the appended packet into `composition_page_evidence`; a shallow copy
+ # there would silently make this never match, which halts rather than misfires.
+ if packet is evidence:
+ challenge_seen = True
+ continue
+ if not challenge_seen or str(packet.get("interaction_tool") or "").strip() != CREDENTIAL_FILL_TOOL_NAME:
+ continue
+ selector = str(packet.get("interaction_selector") or "").strip()
+ # An interaction packet carries the post-interaction URL, so it is attributable to the page
+ # it acted on only by the source it recorded.
+ if (page, selector) in targets and _observed_page_key(packet.get("interaction_source_url")) == page:
+ return True
+ return False
+
+
def terminal_challenge_blocker_signal_from_current_page_evidence(
ctx: Any,
*,
@@ -569,6 +673,49 @@ def terminal_challenge_blocker_signal_from_current_page_evidence(
return None
+CURRENT_PAGE_CHALLENGE_ADVISORY_REASON_CODE = "tool_error_current_page_challenge_advisory"
+
+
+def current_page_challenge_advisory_signal(
+ ctx: Any, *, blocked_tool: str, evidence_source: str = "page_evidence"
+) -> CopilotToolBlockerSignal | None:
+ """A fire-once advisory in place of the current-page pre-veto: the model sees the challenge
+ verdict and decides, holding facts this plane cannot see — what it filled and what it holds.
+
+ Terminal stopping stays with the producers that key on outcomes: a run that failed with
+ anti-bot evidence, the loop plane, budget. A page that merely looks like a wall has not
+ stopped anything yet.
+ """
+ signal = terminal_challenge_blocker_signal_from_current_page_evidence(
+ ctx, blocked_tool=blocked_tool, evidence_source=evidence_source
+ )
+ if signal is None:
+ return None
+ reason = str(signal.extra.get("evidence_reason") or "a verification challenge")
+ fired = getattr(ctx, "challenge_advisory_fired_reasons", None)
+ if not isinstance(fired, set) or reason in fired:
+ return None
+ fired.add(reason)
+ return CopilotToolBlockerSignal(
+ blocker_kind="tool_error",
+ agent_steering_text=(
+ f"Structured page evidence flags the current page as a verification challenge: {reason}. "
+ "This is advisory, not a wall. If the page is asking for something this turn already holds — "
+ "for example a one-time code from the resolved credential — fill it, submit it, and continue. "
+ "If it is a genuine anti-bot wall you cannot satisfy, do NOT retry the same path; reply to the "
+ "user naming the blocker."
+ ),
+ user_facing_reason="The page may be showing a verification step; I'm checking whether I can complete it.",
+ recovery_hint="retry_with_different_tool",
+ cleared_by_tools=frozenset({blocked_tool}),
+ preserves_workflow_draft=True,
+ renders_final_reply=False,
+ internal_reason_code=CURRENT_PAGE_CHALLENGE_ADVISORY_REASON_CODE,
+ blocked_tool=blocked_tool,
+ extra={key: value for key, value in signal.extra.items() if key != "run_outcome_reason_code"},
+ )
+
+
def _maybe_stash_terminal_challenge_halt(ctx: Any) -> None:
if getattr(ctx, "turn_halt", None) is not None:
return
diff --git a/skyvern/forge/sdk/copilot/mcp_adapter.py b/skyvern/forge/sdk/copilot/mcp_adapter.py
index 635a20837..cf13a1f2f 100644
--- a/skyvern/forge/sdk/copilot/mcp_adapter.py
+++ b/skyvern/forge/sdk/copilot/mcp_adapter.py
@@ -36,10 +36,10 @@ from skyvern.forge.sdk.copilot.blocker_signal import (
)
from skyvern.forge.sdk.copilot.build_phase import _phase_blocker_signal
from skyvern.forge.sdk.copilot.enforcement import (
+ current_page_challenge_advisory_signal,
register_no_progress_interaction_click,
requested_output_paths_for_derivation,
synthesized_block_persistence_signal,
- terminal_challenge_blocker_signal_from_current_page_evidence,
)
from skyvern.forge.sdk.copilot.loop_detection import (
detect_failed_tool_step_loop_for_ctx,
@@ -144,7 +144,7 @@ class SchemaOverlay:
LOG = structlog.get_logger()
_INTERNAL_TOOL_ARG_KEYS = frozenset({"_summarized"})
-_CURRENT_PAGE_TERMINAL_CHALLENGE_MCP_TOOLS = frozenset(
+_CURRENT_PAGE_CHALLENGE_ADVISORY_MCP_TOOLS = frozenset(
{
"click",
"evaluate",
@@ -165,17 +165,11 @@ def _stash_and_emit_loop_blocker(ctx: Any, loop_message: str, tool_name: str) ->
return payload
-def _stash_and_emit_current_page_terminal_challenge_blocker(ctx: Any, tool_name: str) -> str | None:
- signal = terminal_challenge_blocker_signal_from_current_page_evidence(
- ctx,
- blocked_tool=tool_name,
- evidence_source="mcp_page_evidence",
- )
+def _emit_current_page_challenge_advisory(ctx: Any, tool_name: str) -> str | None:
+ signal = current_page_challenge_advisory_signal(ctx, blocked_tool=tool_name, evidence_source="mcp_page_evidence")
if signal is None:
return None
- payload = emit_blocker_signal_payload(ctx, signal)
- stash_turn_halt_from_blocker_signal(ctx, signal, source="mcp_current_page_terminal_challenge")
- return payload
+ return emit_blocker_signal_payload(ctx, signal)
def _requested_output_path_choices(schema: dict[str, Any], paths: list[str]) -> dict[str, Any]:
@@ -501,17 +495,14 @@ class SkyvernOverlayMCPServer(MCPServer):
return _copilot_to_call_tool_result({"ok": False, "error": payload})
refresh_held_loop_blocker_evidence(copilot_ctx)
- if tool_name in _CURRENT_PAGE_TERMINAL_CHALLENGE_MCP_TOOLS:
- terminal_challenge_payload = _stash_and_emit_current_page_terminal_challenge_blocker(
- copilot_ctx,
- tool_name,
- )
- if terminal_challenge_payload is not None:
- LOG.warning(
- "Current page terminal challenge detected, skipping MCP browser tool",
+ if tool_name in _CURRENT_PAGE_CHALLENGE_ADVISORY_MCP_TOOLS:
+ challenge_advisory_payload = _emit_current_page_challenge_advisory(copilot_ctx, tool_name)
+ if challenge_advisory_payload is not None:
+ LOG.info(
+ "Current page challenge advisory issued for MCP browser tool",
tool_name=tool_name,
)
- return _copilot_to_call_tool_result({"ok": False, "error": terminal_challenge_payload})
+ return _copilot_to_call_tool_result({"ok": False, "error": challenge_advisory_payload})
persistence_signal = synthesized_block_persistence_signal(copilot_ctx, tool_name, arguments)
if persistence_signal is not None:
diff --git a/skyvern/forge/sdk/copilot/runtime.py b/skyvern/forge/sdk/copilot/runtime.py
index 81ea926e1..a75e7bfd1 100644
--- a/skyvern/forge/sdk/copilot/runtime.py
+++ b/skyvern/forge/sdk/copilot/runtime.py
@@ -396,6 +396,8 @@ class AgentContext:
# while scouting the goal path, each tagged with how that state was reached.
# Feeds the per-acted-page composition gate; never persisted into workflow YAML.
flow_evidence: list[dict[str, Any]] = field(default_factory=list)
+ # Challenge-advisory reasons already surfaced to the model this turn, so the advisory fires once.
+ challenge_advisory_fired_reasons: set[str] = field(default_factory=set)
pending_browser_interaction_observation: PendingBrowserInteractionObservation | None = None
# In-turn side channel from workflow mutation calls: block label -> flow_evidence
# observation step used to ground the newly authored page-acting block.
diff --git a/skyvern/forge/sdk/copilot/tools/blockers.py b/skyvern/forge/sdk/copilot/tools/blockers.py
index bd73aef42..9c4ef5c40 100644
--- a/skyvern/forge/sdk/copilot/tools/blockers.py
+++ b/skyvern/forge/sdk/copilot/tools/blockers.py
@@ -26,8 +26,9 @@ from skyvern.forge.sdk.copilot.completion_verification import (
)
from skyvern.forge.sdk.copilot.composition_evidence import interactive_challenge_controls
from skyvern.forge.sdk.copilot.enforcement import (
+ current_page_challenge_advisory_signal,
+ one_time_code_fill_supersedes_challenge,
synthesized_block_persistence_signal,
- terminal_challenge_blocker_signal_from_current_page_evidence,
)
from skyvern.forge.sdk.copilot.failure_tracking import (
PER_TOOL_BUDGET_FAILURE_CATEGORY,
@@ -64,7 +65,7 @@ from ._shared import (
LOG = structlog.get_logger()
-_CURRENT_PAGE_TERMINAL_CHALLENGE_TOOLS = (
+_CURRENT_PAGE_CHALLENGE_ADVISORY_TOOLS = (
BLOCK_RUNNING_TOOLS
| PAGE_INSPECTION_TOOLS
| frozenset({"click", "navigate_browser", "press_key", "scroll", "select_option", "type_text"})
@@ -667,6 +668,8 @@ def _post_budget_terminal_challenge_signal(
reason = _post_run_terminal_challenge_reason(evidence)
if not reason:
return None
+ if one_time_code_fill_supersedes_challenge(ctx, evidence):
+ return None
# Same-packet adjudication: when the observed page also carries result
# evidence, the answer-from-observed-page path outranks a terminal claim.
@@ -725,14 +728,10 @@ def _post_budget_terminal_challenge_signal(
return _terminal_challenge_signal(reason=reason, arguments=arguments, tool_name=tool_name)
-def _current_page_terminal_challenge_signal(
+def _current_page_challenge_advisory(
ctx: AgentContext, arguments: dict[str, Any] | None, tool_name: str
) -> CopilotToolBlockerSignal | None:
- signal = terminal_challenge_blocker_signal_from_current_page_evidence(
- ctx,
- blocked_tool=tool_name,
- evidence_source="page_evidence",
- )
+ signal = current_page_challenge_advisory_signal(ctx, blocked_tool=tool_name)
if signal is None:
return None
requested_labels = _requested_block_label_set(arguments)
@@ -1369,6 +1368,9 @@ def _challenge_gated_anti_bot_rerun_signal(
) -> CopilotToolBlockerSignal | None:
if not _last_run_has_terminal_anti_bot_blocker(ctx):
return None
+ evidence = getattr(ctx, "composition_page_evidence", None)
+ if isinstance(evidence, dict) and one_time_code_fill_supersedes_challenge(ctx, evidence):
+ return None
if tool_name == "update_and_run_blocks" and _challenge_gated_proxy_retry_allowed(ctx, arguments):
return None
@@ -1411,6 +1413,7 @@ LOOP_PLANE_REFUSAL_REASON_CODES: frozenset[str] = frozenset(
"code_authoring_guardrail_churn",
"credential_priority_authoring_churn",
"tool_error_terminal_challenge_blocker",
+ "tool_error_current_page_challenge_advisory",
"tool_error_synthesized_block_persistence_required",
"tool_error_challenge_gated_submit_disabled",
"tool_error_non_retriable_nav",
@@ -1440,10 +1443,10 @@ def _emit_loop_plane_refusal(ctx: AgentContext, signal: CopilotToolBlockerSignal
def _tool_loop_error(ctx: AgentContext, tool_name: str, arguments: dict[str, Any] | None = None) -> str | None:
refresh_held_loop_blocker_evidence(ctx)
- if tool_name in _CURRENT_PAGE_TERMINAL_CHALLENGE_TOOLS:
- current_page_challenge_signal = _current_page_terminal_challenge_signal(ctx, arguments, tool_name)
- if current_page_challenge_signal is not None:
- return _emit_loop_plane_refusal(ctx, current_page_challenge_signal)
+ if tool_name in _CURRENT_PAGE_CHALLENGE_ADVISORY_TOOLS:
+ challenge_advisory = _current_page_challenge_advisory(ctx, arguments, tool_name)
+ if challenge_advisory is not None:
+ return _emit_loop_plane_refusal(ctx, challenge_advisory)
# The diagnosis contract's STOP only runs BETWEEN agent turns, so without this entrypoint
# check the model can re-author around an unreachable sandbox before the nudge ever fires.
diff --git a/tests/unit/test_copilot_loop_detection_blocker.py b/tests/unit/test_copilot_loop_detection_blocker.py
index 458e985ba..22d56c1f4 100644
--- a/tests/unit/test_copilot_loop_detection_blocker.py
+++ b/tests/unit/test_copilot_loop_detection_blocker.py
@@ -15,14 +15,16 @@ from skyvern.forge.sdk.copilot.blocker_signal import (
)
from skyvern.forge.sdk.copilot.config import BlockAuthoringPolicy
from skyvern.forge.sdk.copilot.context import CopilotContext
-from skyvern.forge.sdk.copilot.enforcement import SYNTHESIZED_BLOCK_PERSISTENCE_REASON_CODE
+from skyvern.forge.sdk.copilot.enforcement import (
+ CURRENT_PAGE_CHALLENGE_ADVISORY_REASON_CODE,
+ SYNTHESIZED_BLOCK_PERSISTENCE_REASON_CODE,
+)
from skyvern.forge.sdk.copilot.loop_detection import tool_step_identity
from skyvern.forge.sdk.copilot.mcp_adapter import SchemaOverlay, SkyvernOverlayMCPServer, _stash_and_emit_loop_blocker
from skyvern.forge.sdk.copilot.output_policy import CopilotOutputKind, evaluate_output_policy
-from skyvern.forge.sdk.copilot.run_outcome import TERMINAL_CHALLENGE_BLOCKER_REASON_CODE, RecordedRunOutcome
+from skyvern.forge.sdk.copilot.run_outcome import RecordedRunOutcome
from skyvern.forge.sdk.copilot.tools import _build_loop_blocker_signal, _tool_loop_error
from skyvern.forge.sdk.copilot.tools.mcp_hooks import get_skyvern_mcp_alias_map
-from skyvern.forge.sdk.copilot.turn_halt import TurnHaltKind
from skyvern.forge.sdk.copilot.turn_intent import TurnIntent, TurnIntentAuthority, TurnIntentMode
_LEAK_TOKENS = ("safe_reason_code", "LOOP DETECTED:")
@@ -355,7 +357,7 @@ def test_fixed_tier_copy_is_clean_for_every_loop_prone_tool(blocked_tool: str, r
assert_clean_user_facing_text(signal.user_facing_reason, blocked_tool=blocked_tool)
-def test_native_tool_loop_error_terminal_challenge_preempts_same_tool_loop() -> None:
+def test_native_tool_loop_error_challenge_advisory_fires_once_then_loop_plane_owns() -> None:
ctx = _ctx(consecutive_tool_tracker=_streak("evaluate", {"expression": "document.body.innerText"}))
ctx.composition_page_evidence = _current_page_challenge_evidence()
@@ -364,21 +366,23 @@ def test_native_tool_loop_error_terminal_challenge_preempts_same_tool_loop() ->
assert payload is not None
signal = ctx.blocker_signal
assert isinstance(signal, CopilotToolBlockerSignal)
- assert signal.internal_reason_code == TERMINAL_CHALLENGE_BLOCKER_REASON_CODE
+ assert signal.internal_reason_code == CURRENT_PAGE_CHALLENGE_ADVISORY_REASON_CODE
assert signal.blocked_tool == "evaluate"
assert signal.extra["evidence_source"] == "page_evidence"
- assert ctx.turn_halt is not None
- assert ctx.turn_halt.kind == TurnHaltKind.ACTIVE_TERMINAL_CHALLENGE
+ assert signal.renders_final_reply is False
+ assert ctx.turn_halt is None
+
+ assert _tool_loop_error(ctx, "evaluate", {"expression": "document.body.innerText"}) is None
@pytest.mark.asyncio
-async def test_mcp_browser_tool_terminal_challenge_preempts_same_tool_loop() -> None:
+async def test_mcp_browser_tool_challenge_advisory_defers_execution_once() -> None:
ctx = _ctx(consecutive_tool_tracker=_streak("evaluate", {"expression": "document.body.innerText"}))
ctx.composition_page_evidence = _current_page_challenge_evidence()
class _UnexpectedClient:
async def call_tool(self, name: str, args: dict, raise_on_error: bool = False) -> object:
- raise AssertionError("terminal challenge blocker should skip MCP execution")
+ raise AssertionError("the challenge advisory should defer MCP execution")
server = SkyvernOverlayMCPServer(
transport=None,
@@ -396,11 +400,35 @@ async def test_mcp_browser_tool_terminal_challenge_preempts_same_tool_loop() ->
assert parsed["ok"] is False
signal = ctx.blocker_signal
assert isinstance(signal, CopilotToolBlockerSignal)
- assert signal.internal_reason_code == TERMINAL_CHALLENGE_BLOCKER_REASON_CODE
+ assert signal.internal_reason_code == CURRENT_PAGE_CHALLENGE_ADVISORY_REASON_CODE
assert signal.blocked_tool == "evaluate"
assert signal.extra["evidence_source"] == "mcp_page_evidence"
- assert ctx.turn_halt is not None
- assert ctx.turn_halt.kind == TurnHaltKind.ACTIVE_TERMINAL_CHALLENGE
+ assert ctx.turn_halt is None
+
+
+async def test_mcp_browser_tool_executes_after_the_challenge_advisory_was_delivered() -> None:
+ ctx = _ctx()
+ ctx.composition_page_evidence = _current_page_challenge_evidence()
+
+ calls: list[str] = []
+
+ class _CountingClient:
+ async def call_tool(self, name: str, args: dict, raise_on_error: bool = False) -> _OkResult:
+ calls.append(name)
+ return _OkResult()
+
+ server = _dispatch_server(ctx, _CountingClient())
+
+ first = await server.call_tool("evaluate", {"expression": "document.title"})
+ assert first.isError is True
+ assert calls == []
+ assert ctx.turn_halt is None
+
+ second = await server.call_tool("evaluate", {"expression": "document.title"})
+ assert second.isError is False
+ assert calls == ["evaluate"]
+ assert ctx.turn_halt is None
@pytest.mark.asyncio
diff --git a/tests/unit/test_copilot_per_tool_budget.py b/tests/unit/test_copilot_per_tool_budget.py
index d91070253..c18f6a63b 100644
--- a/tests/unit/test_copilot_per_tool_budget.py
+++ b/tests/unit/test_copilot_per_tool_budget.py
@@ -32,6 +32,7 @@ from skyvern.forge.sdk.copilot.diagnosis_repair_contract import (
VerificationResult,
)
from skyvern.forge.sdk.copilot.enforcement import (
+ CURRENT_PAGE_CHALLENGE_ADVISORY_REASON_CODE,
MAX_PER_TOOL_BUDGET_NUDGES,
POST_ANTI_BOT_FAILED_TEST_NUDGE,
POST_FAILED_TEST_NUDGE,
@@ -44,10 +45,6 @@ from skyvern.forge.sdk.copilot.failure_tracking import (
PER_TOOL_BUDGET_FAILURE_CATEGORY,
compute_failure_signature,
)
-from skyvern.forge.sdk.copilot.run_outcome import (
- TERMINAL_CHALLENGE_BLOCKER_REASON_CODE,
- TERMINAL_CHALLENGE_RUN_OUTCOME_REASON_CODE,
-)
from skyvern.forge.sdk.copilot.tools import (
WatchdogExitReason,
_composition_anti_bot_reason,
@@ -64,6 +61,7 @@ from skyvern.forge.sdk.copilot.tools import (
_tool_loop_error,
_watchdog_user_failure_reason,
)
+from skyvern.forge.sdk.copilot.tools.blockers import _challenge_gated_anti_bot_rerun_signal
from skyvern.forge.sdk.copilot.turn_halt import CopilotTurnHalt, TurnHaltKind
from tests.unit.conftest import make_copilot_context as _fresh_context
@@ -159,7 +157,7 @@ def test_composition_anti_bot_reason_reads_typed_challenge_state_without_legacy_
assert "challenge-gated disabled submit/search control: Search" in reason
-def test_block_running_halts_on_current_page_terminal_challenge_before_repair_loop() -> None:
+def test_block_running_gets_the_challenge_advisory_before_repair_loop() -> None:
ctx = _fresh_context()
ctx.composition_page_evidence = {
"observed_after_workflow_run": True,
@@ -176,12 +174,11 @@ def test_block_running_halts_on_current_page_terminal_challenge_before_repair_lo
assert msg is not None
assert ctx.blocker_signal is not None
- assert ctx.blocker_signal.internal_reason_code == TERMINAL_CHALLENGE_BLOCKER_REASON_CODE
- assert ctx.blocker_signal.extra["run_outcome_reason_code"] == TERMINAL_CHALLENGE_RUN_OUTCOME_REASON_CODE
+ assert ctx.blocker_signal.internal_reason_code == CURRENT_PAGE_CHALLENGE_ADVISORY_REASON_CODE
assert ctx.blocker_signal.extra["evidence_source"] == "page_evidence"
assert ctx.blocker_signal.extra["block_labels"] == ["search_lookup"]
- assert ctx.turn_halt is not None
- assert ctx.turn_halt.kind == TurnHaltKind.ACTIVE_TERMINAL_CHALLENGE
+ assert ctx.turn_halt is None
+ assert _tool_loop_error(ctx, "update_and_run_blocks", {"block_labels": ["search_lookup"]}) is None
def test_block_running_does_not_halt_current_page_challenge_when_results_are_present() -> None:
@@ -202,7 +199,7 @@ def test_block_running_does_not_halt_current_page_challenge_when_results_are_pre
assert ctx.turn_halt is None
-def test_block_running_halts_current_page_challenge_with_empty_result_shell() -> None:
+def test_block_running_gets_the_challenge_advisory_on_an_empty_result_shell() -> None:
ctx = _fresh_context()
ctx.composition_page_evidence = {
"observed_after_workflow_run": True,
@@ -220,9 +217,8 @@ def test_block_running_halts_current_page_challenge_with_empty_result_shell() ->
assert msg is not None
assert ctx.blocker_signal is not None
- assert ctx.blocker_signal.internal_reason_code == TERMINAL_CHALLENGE_BLOCKER_REASON_CODE
- assert ctx.turn_halt is not None
- assert ctx.turn_halt.kind == TurnHaltKind.ACTIVE_TERMINAL_CHALLENGE
+ assert ctx.blocker_signal.internal_reason_code == CURRENT_PAGE_CHALLENGE_ADVISORY_REASON_CODE
+ assert ctx.turn_halt is None
def test_block_running_after_budget_current_url_requires_inspection_first() -> None:
@@ -712,7 +708,7 @@ def test_tool_loop_error_allows_new_smaller_label_after_budget_problem() -> None
)
-def test_tool_loop_error_blocks_challenge_gated_disabled_submit_rerun() -> None:
+def test_challenge_advisory_then_attempt_keyed_rerun_stop_on_unchanged_path() -> None:
ctx = _fresh_context()
ctx.workflow_yaml = """
title: Registry standard credential lookup
@@ -749,13 +745,27 @@ workflow_definition:
)
assert msg is not None
- assert "Structured challenge evidence confirms this path is blocked" in msg
assert ctx.blocker_signal is not None
- assert ctx.blocker_signal.internal_reason_code == TERMINAL_CHALLENGE_BLOCKER_REASON_CODE
- assert ctx.blocker_signal.renders_final_reply is True
+ assert ctx.blocker_signal.internal_reason_code == CURRENT_PAGE_CHALLENGE_ADVISORY_REASON_CODE
+ assert ctx.blocker_signal.renders_final_reply is False
+ assert ctx.turn_halt is None
+ repeat = _tool_loop_error(
+ ctx,
+ "update_and_run_blocks",
+ {
+ "workflow_yaml": ctx.workflow_yaml,
+ "block_labels": ["fill_sample_record_search"],
+ },
+ )
-def test_tool_loop_error_blocks_current_page_challenge_with_changed_proxy_location() -> None:
+ assert repeat is not None
+ assert ctx.latest_tool_blocker_signal is not None
+ assert ctx.latest_tool_blocker_signal.internal_reason_code == "tool_error_challenge_gated_submit_disabled"
+ assert ctx.latest_tool_blocker_signal.renders_final_reply is True
+
+
+def test_challenge_advisory_then_single_proxy_retry_then_attempt_keyed_stop() -> None:
ctx = _fresh_context()
ctx.last_failed_workflow_yaml = """
title: Registry standard credential lookup
@@ -798,10 +808,36 @@ workflow_definition:
)
assert msg is not None
- assert "do NOT try a proxy/location switch" in msg
- assert ctx.challenge_gated_proxy_retry_count == 0
assert ctx.blocker_signal is not None
- assert ctx.blocker_signal.internal_reason_code == TERMINAL_CHALLENGE_BLOCKER_REASON_CODE
+ assert ctx.blocker_signal.internal_reason_code == CURRENT_PAGE_CHALLENGE_ADVISORY_REASON_CODE
+ assert ctx.turn_halt is None
+
+ proxy_retry = _tool_loop_error(
+ ctx,
+ "update_and_run_blocks",
+ {
+ "workflow_yaml": changed_proxy_yaml,
+ "block_labels": ["fill_sample_record_search"],
+ },
+ )
+
+ assert proxy_retry is None
+
+ exhausted = _tool_loop_error(
+ ctx,
+ "update_and_run_blocks",
+ {
+ "workflow_yaml": changed_proxy_yaml,
+ "block_labels": ["fill_sample_record_search"],
+ },
+ )
+
+ assert exhausted is not None
+ assert ctx.latest_tool_blocker_signal is not None
+ assert ctx.latest_tool_blocker_signal.internal_reason_code == "tool_error_challenge_gated_submit_disabled"
+ assert ctx.challenge_gated_proxy_retry_count == 1
+ assert ctx.blocker_signal is not None
+ assert ctx.blocker_signal.internal_reason_code == CURRENT_PAGE_CHALLENGE_ADVISORY_REASON_CODE
def test_tool_loop_error_blocks_post_budget_challenge_even_with_changed_proxy_location() -> None:
@@ -1635,3 +1671,113 @@ def test_rendered_challenge_widget_still_produces_terminal_reason() -> None:
reason = _post_run_terminal_challenge_reason(evidence)
assert reason is not None
assert "disabled" in reason
+
+
+_CODE_PAGE_URL = "https://example.com/account/login/totp?next=%2Freports"
+
+
+def _code_page_challenge_evidence() -> dict:
+ """A challenge on a one-time-code page, of a kind a saved code can answer."""
+ return _challenge_gated_post_run_evidence(
+ current_url=_CODE_PAGE_URL,
+ challenge_state={
+ "detected": True,
+ "kind": "other",
+ "requires_human_verification": True,
+ "gates_submit_controls": True,
+ "gated_submit_controls": [{"text": "Next", "disabled": True}],
+ },
+ )
+
+
+def _arm_code_already_filled(ctx: CopilotContext, evidence: dict) -> None:
+ """Put the challenge in flow evidence, then a one-time-code fill observed after it."""
+ ctx.composition_page_evidence = evidence
+ ctx.scout_trajectory = [
+ {
+ "tool_name": "fill_credential_field",
+ "selector": "#totp",
+ "source_url": _CODE_PAGE_URL,
+ "credential_field": "totp",
+ }
+ ]
+ ctx.flow_evidence = [
+ {"evidence": evidence},
+ {
+ "evidence": {
+ "current_url": _CODE_PAGE_URL,
+ "interaction_tool": "fill_credential_field",
+ "interaction_selector": "#totp",
+ "interaction_source_url": _CODE_PAGE_URL,
+ }
+ },
+ ]
+
+
+def _arm_challenge_gated_rerun(ctx: CopilotContext) -> None:
+ ctx.last_test_anti_bot = "challenge-gated disabled submit/search control"
+ ctx.last_test_failure_reason = "blocker: the submit control was disabled by a verification challenge"
+
+
+def test_post_budget_challenge_declines_a_code_page_the_turn_already_answered() -> None:
+ ctx = _armed_post_run_context()
+ _arm_code_already_filled(ctx, _code_page_challenge_evidence())
+
+ assert _post_budget_terminal_challenge_signal(ctx, _BLOCK_LABEL_ARGS, "update_and_run_blocks") is None
+
+
+def test_post_budget_challenge_still_fires_without_a_one_time_code_fill() -> None:
+ ctx = _armed_post_run_context()
+ ctx.composition_page_evidence = _code_page_challenge_evidence()
+
+ assert _post_budget_terminal_challenge_signal(ctx, _BLOCK_LABEL_ARGS, "update_and_run_blocks") is not None
+
+
+def test_challenge_gated_rerun_declines_a_code_page_the_turn_already_answered() -> None:
+ ctx = _armed_post_run_context()
+ _arm_challenge_gated_rerun(ctx)
+ _arm_code_already_filled(ctx, _code_page_challenge_evidence())
+
+ assert _challenge_gated_anti_bot_rerun_signal(ctx, _BLOCK_LABEL_ARGS, "get_run_results") is None
+
+
+def test_challenge_gated_rerun_still_fires_without_a_one_time_code_fill() -> None:
+ ctx = _armed_post_run_context()
+ _arm_challenge_gated_rerun(ctx)
+ ctx.composition_page_evidence = _code_page_challenge_evidence()
+
+ assert _challenge_gated_anti_bot_rerun_signal(ctx, _BLOCK_LABEL_ARGS, "get_run_results") is not None
+
+
+def test_challenge_gated_rerun_still_fires_when_no_page_evidence_was_captured() -> None:
+ """With no page observation there is nothing to order the fill against, so the stop stands."""
+ ctx = _armed_post_run_context()
+ _arm_challenge_gated_rerun(ctx)
+ ctx.last_test_failure_reason = "blocker: bot detection challenge interrupted the run"
+ ctx.composition_page_evidence = None
+ ctx.scout_trajectory = [
+ {
+ "tool_name": "fill_credential_field",
+ "selector": "#totp",
+ "source_url": _CODE_PAGE_URL,
+ "credential_field": "totp",
+ }
+ ]
+
+ assert _challenge_gated_anti_bot_rerun_signal(ctx, _BLOCK_LABEL_ARGS, "get_run_results") is not None
+
+
+def test_challenge_gated_rerun_still_fires_on_a_benign_packet_after_a_code_fill() -> None:
+ """A run-reported bot wall is not answered by a code typed into an unchallenged page."""
+ ctx = _armed_post_run_context()
+ _arm_challenge_gated_rerun(ctx)
+ ctx.last_test_failure_reason = "blocker: bot detection challenge interrupted the run"
+ benign = {
+ "current_url": _CODE_PAGE_URL,
+ "inspected_url": _CODE_PAGE_URL,
+ "observed_after_workflow_run": True,
+ "forms": [],
+ }
+ _arm_code_already_filled(ctx, benign)
+
+ assert _challenge_gated_anti_bot_rerun_signal(ctx, _BLOCK_LABEL_ARGS, "get_run_results") is not None
diff --git a/tests/unit/test_copilot_turn_halt.py b/tests/unit/test_copilot_turn_halt.py
index 9d326bf90..ef871788d 100644
--- a/tests/unit/test_copilot_turn_halt.py
+++ b/tests/unit/test_copilot_turn_halt.py
@@ -195,9 +195,248 @@ def test_page_challenge_signal_records_explicit_terminal_blocker() -> None:
assert signal.blocked_tool == "update_and_run_blocks"
+_TOTP_URL = "https://portal.example.com/account/login/totp?next=%2Freports"
+_LOGIN_URL = "https://portal.example.com/account/login?next=%2Freports"
+_OTHER_TOTP_URL = "https://other.example.com/account/login/totp"
+
+
+def _vision_challenge_packet() -> dict[str, object]:
+ """A vision-stamped verification verdict on the one-time-code page."""
+ return {
+ "current_url": _TOTP_URL,
+ "inspected_url": _TOTP_URL,
+ "observed_after_workflow_run": True,
+ "challenge_state": {
+ "detected": True,
+ "kind": "other",
+ "requires_human_verification": True,
+ "evidence_source": "vision",
+ },
+ }
+
+
+def _captcha_widget_packet() -> dict[str, object]:
+ """A rendered challenge widget on the same page: a control the copilot must actually solve."""
+ return {
+ "current_url": _TOTP_URL,
+ "inspected_url": _TOTP_URL,
+ "observed_after_workflow_run": True,
+ "challenge_controls": [{"kind": "recaptcha", "selector": "iframe[title='reCAPTCHA']", "visible": True}],
+ }
+
+
+def _captcha_kind_packet() -> dict[str, object]:
+ """A vision-confirmed CAPTCHA carrying no parsed DOM control — named by kind alone."""
+ packet = _vision_challenge_packet()
+ challenge_state = packet["challenge_state"]
+ assert isinstance(challenge_state, dict)
+ challenge_state["kind"] = "captcha"
+ return packet
+
+
+def _unknown_kind_packet() -> dict[str, object]:
+ """The DOM detector's verdict for an anti-bot vendor it has no name for."""
+ packet = _vision_challenge_packet()
+ challenge_state = packet["challenge_state"]
+ assert isinstance(challenge_state, dict)
+ challenge_state["kind"] = "unknown"
+ return packet
+
+
+def _interaction_packet(
+ tool: str, selector: str, *, landed_url: str = _TOTP_URL, acted_url: str = _TOTP_URL
+) -> dict[str, object]:
+ return {
+ "current_url": landed_url,
+ "inspected_url": landed_url,
+ "source_tool": "scout_interaction",
+ "interaction_tool": tool,
+ "interaction_selector": selector,
+ "interaction_source_url": acted_url,
+ }
+
+
+def _interaction_challenge_packet(tool: str, selector: str) -> dict[str, object]:
+ """A click's own post-action observation, which merges the parsed page evidence."""
+ return {**_vision_challenge_packet(), **_interaction_packet(tool, selector)}
+
+
+def _credential_login_ctx(
+ *,
+ code_filled: bool,
+ code_submitted: bool = False,
+ challenge_reobserved_before_submit: bool = False,
+ challenge_reobserved_after_submit: bool = False,
+ challenge_packet: dict[str, object] | None = None,
+ intervening_click: bool = False,
+ submit_navigated: bool = False,
+ enter_submitted: bool = False,
+ fill_on_another_page: bool = False,
+ fill_acted_elsewhere: bool = False,
+) -> SimpleNamespace:
+ trajectory: list[dict[str, object]] = [
+ {
+ "tool_name": "fill_credential_field",
+ "selector": "#username",
+ "source_url": _LOGIN_URL,
+ "credential_field": "username",
+ },
+ {
+ "tool_name": "fill_credential_field",
+ "selector": "#password",
+ "source_url": _LOGIN_URL,
+ "credential_field": "password",
+ },
+ {"tool_name": "click", "selector": 'button[aria-label="Log in"]', "source_url": _LOGIN_URL},
+ ]
+ flow_evidence: list[dict[str, object]] = [{"evidence": challenge_packet or _vision_challenge_packet()}]
+ if code_filled:
+ fill_page = _OTHER_TOTP_URL if fill_on_another_page else _TOTP_URL
+ trajectory.append(
+ {
+ "tool_name": "fill_credential_field",
+ "selector": "#totp",
+ "source_url": fill_page,
+ "credential_field": "totp",
+ }
+ )
+ # A packet whose recorded source is elsewhere must not be credited to the challenged page,
+ # even though it landed there and its selector matches.
+ acted = _OTHER_TOTP_URL if fill_acted_elsewhere else fill_page
+ flow_evidence.append({"evidence": _interaction_packet("fill_credential_field", "#totp", acted_url=acted)})
+ if challenge_reobserved_before_submit:
+ flow_evidence.append({"evidence": _vision_challenge_packet()})
+ if intervening_click:
+ trajectory.append({"tool_name": "click", "selector": "#cookie-accept", "source_url": _TOTP_URL})
+ flow_evidence.append({"evidence": _interaction_challenge_packet("click", "#cookie-accept")})
+ if code_submitted and enter_submitted:
+ # An Enter keypress commits the code without minting an interaction observation.
+ trajectory.append({"tool_name": "press_key", "key": "Enter", "source_url": _TOTP_URL})
+ elif code_submitted:
+ trajectory.append({"tool_name": "click", "selector": "#totp-submit", "source_url": _TOTP_URL})
+ landed = "https://portal.example.com/reports" if submit_navigated else _TOTP_URL
+ flow_evidence.append({"evidence": _interaction_packet("click", "#totp-submit", landed_url=landed)})
+ if challenge_reobserved_after_submit:
+ flow_evidence.append({"evidence": _vision_challenge_packet()})
+ return SimpleNamespace(
+ flow_evidence=flow_evidence,
+ scout_trajectory=trajectory,
+ last_failure_category_top=None,
+ last_run_blocks_workflow_run_id="wr_000000000000000000",
+ )
+
+
+def _backstop_ctx(*, code_filled: bool) -> SimpleNamespace:
+ login = _credential_login_ctx(code_filled=code_filled)
+ return _halt_ctx(
+ flow_evidence=login.flow_evidence,
+ scout_trajectory=login.scout_trajectory,
+ composition_page_evidence=login.flow_evidence[0]["evidence"],
+ last_run_outcome=None,
+ last_test_ok=False,
+ last_test_anti_bot=True,
+ )
+
+
+ ("code_filled", "expect_halt"),
+ [(True, False), (False, True)],
+ ids=["backstop_declines_the_stale_packet_the_code_answered", "backstop_still_halts_without_a_code_fill"],
+)
+def test_finalize_backstop_shares_the_credential_served_decline(code_filled: bool, expect_halt: bool) -> None:
+ ctx = _backstop_ctx(code_filled=code_filled)
+
+ _maybe_stash_terminal_challenge_halt(ctx)
+
+ assert (ctx.turn_halt is not None) is expect_halt
+
+
@pytest.mark.parametrize(
("ctx", "blocked_tool", "expect_signal"),
[
+ pytest.param(
+ _credential_login_ctx(code_filled=True),
+ "click",
+ False,
+ id="declines_on_a_challenge_seen_before_the_code_was_filled",
+ ),
+ pytest.param(
+ _credential_login_ctx(code_filled=True, code_submitted=True),
+ "evaluate",
+ False,
+ id="stale_challenge_stays_declined_for_the_tool_after_the_submit",
+ ),
+ pytest.param(
+ _credential_login_ctx(code_filled=True, challenge_reobserved_before_submit=True),
+ "click",
+ True,
+ id="halts_on_a_challenge_observed_after_the_fill_even_before_a_submit",
+ ),
+ pytest.param(
+ _credential_login_ctx(code_filled=True, code_submitted=True, enter_submitted=True),
+ "evaluate",
+ False,
+ id="an_enter_submit_cannot_extend_the_decline_past_the_fill",
+ ),
+ pytest.param(
+ _credential_login_ctx(code_filled=True, fill_on_another_page=True),
+ "click",
+ True,
+ id="halts_when_the_code_was_filled_into_a_different_page",
+ ),
+ pytest.param(
+ _credential_login_ctx(code_filled=True, fill_acted_elsewhere=True),
+ "click",
+ True,
+ id="halts_when_the_fill_packet_acted_on_a_different_page_than_it_landed_on",
+ ),
+ pytest.param(
+ _credential_login_ctx(code_filled=True, challenge_packet=_unknown_kind_packet()),
+ "click",
+ True,
+ id="halts_on_an_unnamed_anti_bot_kind_the_dom_detector_could_not_classify",
+ ),
+ pytest.param(
+ _credential_login_ctx(code_filled=True, code_submitted=True, challenge_reobserved_after_submit=True),
+ "click",
+ True,
+ id="halts_when_the_page_still_demands_a_code_after_the_submit",
+ ),
+ pytest.param(
+ _credential_login_ctx(code_filled=True, intervening_click=True),
+ "click",
+ True,
+ id="halts_when_an_intervening_click_reobserves_the_challenge_itself",
+ ),
+ pytest.param(
+ _credential_login_ctx(
+ code_filled=True,
+ code_submitted=True,
+ submit_navigated=True,
+ challenge_reobserved_after_submit=True,
+ ),
+ "click",
+ True,
+ id="halts_after_a_navigating_submit_when_the_page_still_demands_a_code",
+ ),
+ pytest.param(
+ _credential_login_ctx(code_filled=True, challenge_packet=_captcha_widget_packet()),
+ "click",
+ True,
+ id="halts_on_a_rendered_challenge_widget_the_code_cannot_satisfy",
+ ),
+ pytest.param(
+ _credential_login_ctx(code_filled=True, challenge_packet=_captcha_kind_packet()),
+ "click",
+ True,
+ id="halts_on_a_vision_named_captcha_with_no_dom_control",
+ ),
+ pytest.param(
+ _credential_login_ctx(code_filled=False),
+ "click",
+ True,
+ id="halts_when_no_credential_fill_reached_the_challenged_page",
+ ),
pytest.param(
SimpleNamespace(
last_failure_category_top=None,