Skip to content

Automate browser based workflows with AI

fix(log-noise): downgrade handled screenshot-timeout from error to warning (#7757)

Shuchang Zhengbc396d04758d
diff --git a/skyvern/forge/sdk/copilot/challenge_evidence.py b/skyvern/forge/sdk/copilot/challenge_evidence.py
index a52f6c065..1f722a7c0 100644
--- a/skyvern/forge/sdk/copilot/challenge_evidence.py
+++ b/skyvern/forge/sdk/copilot/challenge_evidence.py
@@ -194,6 +194,39 @@ def composition_challenge_carrier(evidence: Mapping[str, Any] | None) -> Challen
return None
+def _challenge_signalled(evidence: Mapping[str, Any] | None) -> bool:
+ if not isinstance(evidence, Mapping):
+ return False
+ challenge_state = evidence.get("challenge_state")
+ return isinstance(challenge_state, Mapping) and challenge_state.get("detected") is True
+
+
+def challenge_signal_regressed(
+ previous: Mapping[str, Any] | None,
+ candidate: Mapping[str, Any] | None,
+) -> bool:
+ """True when ``candidate`` drops a challenge signal ``previous`` already carried.
+
+ Looking again for a carrier must never erase the signal that justified looking:
+ a signalled packet is the only trigger for the visual fallback, so replacing one
+ with an unsignalled packet removes the last detection channel on this path."""
+ return _challenge_signalled(previous) and not _challenge_signalled(candidate)
+
+
+def challenge_evidence_unsettled(evidence: Mapping[str, Any] | None) -> bool:
+ """True when a challenge was signalled but no carrier has rendered yet, so one
+ more capture may witness it. A widget that renders asynchronously is absent from
+ a capture taken at first paint, and a page holding a form is otherwise treated as
+ fully captured. This decides only whether to look again — never whether a
+ challenge exists, which stays with ``composition_challenge_carrier``."""
+ if not isinstance(evidence, Mapping):
+ return False
+ challenge_state = evidence.get("challenge_state")
+ if not isinstance(challenge_state, Mapping) or challenge_state.get("detected") is not True:
+ return False
+ return composition_challenge_carrier(evidence) is None
+
+
def vision_challenge_carrier(visual_summary: Mapping[str, Any] | None) -> bool:
if not isinstance(visual_summary, Mapping) or visual_summary.get("challenge_detected") is not True:
return False
diff --git a/skyvern/forge/sdk/copilot/runtime.py b/skyvern/forge/sdk/copilot/runtime.py
index d4cee164c..4d1317e6f 100644
--- a/skyvern/forge/sdk/copilot/runtime.py
+++ b/skyvern/forge/sdk/copilot/runtime.py
@@ -283,6 +283,9 @@ class ScoutedInteraction(TypedDict):
element_fingerprint_test_id: NotRequired[str]
element_fingerprint_tag: NotRequired[str]
element_fingerprint_probed: NotRequired[str]
+ # Stamped when exploration met a challenge here; read back by composition_challenge_carrier
+ # so synthesis emits solve_captcha(page) at this boundary.
+ challenge_state: NotRequired[dict[str, Any]]
NeverCapturedObligationState: TypeAlias = Literal["armed", "captured", "consumed"]
@@ -507,6 +510,9 @@ class AgentContext:
# preserves repeats and ordering so code_block_synthesis can emit a faithful
# linear Playwright trajectory.
scout_trajectory: list[ScoutedInteraction] = field(default_factory=list)
+ # Solve attempts already spent per challenge. A challenge that never passes is precisely the
+ # one re-observed on every later capture, so cost otherwise grows with how stuck the turn is.
+ challenge_solve_attempts: dict[str, int] = field(default_factory=dict)
# One exact `never_captured` mutation may reopen scouting within this turn. The obligation is
# completed only by a later generator-emitted canonical interaction, never by selector text alone.
never_captured_obligation: NeverCapturedObligation | None = None
diff --git a/skyvern/forge/sdk/copilot/tools/composition_capture.py b/skyvern/forge/sdk/copilot/tools/composition_capture.py
index 6bb2ca997..f059df799 100644
--- a/skyvern/forge/sdk/copilot/tools/composition_capture.py
+++ b/skyvern/forge/sdk/copilot/tools/composition_capture.py
@@ -10,6 +10,7 @@ from urllib.parse import urlparse
import structlog
from skyvern.forge.sdk.copilot.build_test_outcome import maybe_satisfy_recorded_outcome_grounding_requirement
+from skyvern.forge.sdk.copilot.challenge_evidence import challenge_evidence_unsettled
from skyvern.forge.sdk.copilot.completion_verification import (
CompletionVerificationResult,
RunEvidenceSnapshot,
@@ -65,6 +66,7 @@ from .scouting import (
_consume_pending_browser_interaction_observation,
_mark_page_inspected,
_mark_post_run_page_observed,
+ solve_challenge_when_evidence_settles,
)
LOG = structlog.get_logger()
@@ -122,6 +124,9 @@ async def _active_run_terminal_evidence_sample(
inspected_url=current_url,
current_url=current_url,
active_run_terminal_sample=True,
+ # The executor is still driving this page; solving here would act on a run this turn
+ # does not own, and spend the turn's attempts on a sample.
+ solve_challenges=False,
)
if html_error is not None or evidence is None:
LOG.info(
@@ -608,12 +613,17 @@ async def _augment_composition_evidence_with_computed_obstruction_candidates(
return _merge_visual_obstruction_candidates(evidence, candidates)
+def _composition_capture_settled(evidence: dict[str, Any]) -> bool:
+ return has_bounded_page_schema(evidence) and not challenge_evidence_unsettled(evidence)
+
+
async def _capture_composition_evidence(
copilot_ctx: Any,
*,
inspected_url: str,
current_url: str,
active_run_terminal_sample: bool = False,
+ solve_challenges: bool = True,
) -> tuple[dict[str, Any] | None, str | None]:
"""Parse composition evidence (cheap extractor first, get_html fallback); html_error is set only on a failed HTML read."""
evidence: dict[str, Any] | None = None
@@ -627,11 +637,16 @@ async def _capture_composition_evidence(
if structured is not None:
evidence = structured
used_structured = True
- if has_bounded_page_schema(evidence):
+ if _composition_capture_settled(evidence):
break
if attempt < _COMPOSITION_HOLLOW_RECAPTURE_RETRIES:
await asyncio.sleep(_COMPOSITION_HOLLOW_RECAPTURE_DELAY_SECONDS)
continue
+ # get_html reads body only, so re-parsing cannot see a title-derived challenge signal.
+ # Guard the reparse itself, not one route to it: re-looks may be exhausted, or the
+ # extractor may have blinked on a later attempt while a signalled packet is in hand.
+ if used_structured and challenge_evidence_unsettled(evidence):
+ break
html, html_error, html_truncated, used_stripped = await _composition_get_html(copilot_ctx, skip_raw=skip_raw)
if html_error is not None:
if evidence is not None:
@@ -644,7 +659,7 @@ async def _capture_composition_evidence(
skip_raw = True
evidence = parse_composition_html(html, inspected_url=inspected_url, current_url=current_url)
used_structured = False
- if has_bounded_page_schema(evidence):
+ if _composition_capture_settled(evidence):
break
if attempt < _COMPOSITION_HOLLOW_RECAPTURE_RETRIES:
await asyncio.sleep(_COMPOSITION_HOLLOW_RECAPTURE_DELAY_SECONDS)
@@ -659,6 +674,12 @@ async def _capture_composition_evidence(
or (evidence.get("schema_empty_page") is True and not has_bounded_page_schema(evidence))
):
evidence = await _augment_composition_evidence_with_visual_fallback(copilot_ctx, evidence)
+ # Off for a capture that only reads a run's page: solving there would mutate the page being
+ # diagnosed and spend on a session this turn is not driving.
+ if solve_challenges:
+ await solve_challenge_when_evidence_settles(
+ copilot_ctx, evidence, url=current_url, observed_after_interaction=False
+ )
return evidence, None
diff --git a/skyvern/forge/sdk/copilot/tools/run_execution.py b/skyvern/forge/sdk/copilot/tools/run_execution.py
index 7ae32ccd7..a9842fb19 100644
--- a/skyvern/forge/sdk/copilot/tools/run_execution.py
+++ b/skyvern/forge/sdk/copilot/tools/run_execution.py
@@ -1293,7 +1293,9 @@ async def _capture_and_store_post_run_page(
evidence: dict[str, Any] | None = None
try:
evidence, _ = await asyncio.wait_for(
- _capture_composition_evidence(ctx, inspected_url=current_url, current_url=current_url),
+ _capture_composition_evidence(
+ ctx, inspected_url=current_url, current_url=current_url, solve_challenges=False
+ ),
timeout=_POST_RUN_REPAIR_CAPTURE_TIMEOUT_SECONDS,
)
except Exception:
diff --git a/skyvern/forge/sdk/copilot/tools/scouting.py b/skyvern/forge/sdk/copilot/tools/scouting.py
index c1699ee46..7d840b5bb 100644
--- a/skyvern/forge/sdk/copilot/tools/scouting.py
+++ b/skyvern/forge/sdk/copilot/tools/scouting.py
@@ -19,6 +19,14 @@ from skyvern.forge.sdk.copilot.build_test_outcome import (
recorded_outcome_from_loaded_result_evidence,
recorded_outcome_from_scout_act_observe_hollow,
)
+from skyvern.forge.sdk.copilot.challenge_evidence import (
+ CHALLENGE_EVIDENCE_SOURCE_KEY,
+ ChallengeEvidenceSource,
+ challenge_evidence_unsettled,
+ challenge_signal_regressed,
+ composition_challenge_carrier,
+ interactive_challenge_controls,
+)
from skyvern.forge.sdk.copilot.code_block_synthesis import (
_is_positional_selector,
dynamic_row_evidence_fingerprint,
@@ -76,6 +84,7 @@ from skyvern.forge.sdk.copilot.runtime import (
ScoutedInteraction,
resolve_browser_state_for_context,
)
+from skyvern.forge.sdk.workflow.models.block import CodeBlockCaptchaError, _code_block_solve_captcha_builtin
from ._shared import (
_DISCOVERY_PER_CALL_TIMEOUT_SECONDS,
@@ -793,6 +802,125 @@ def _record_scouted_interaction(
record_reached_terminal_action_observation(ctx)
+_MAX_CHALLENGE_SOLVE_ATTEMPTS = 3
+
+
+def _challenge_identity(evidence: dict[str, Any]) -> str:
+ """Identify the challenge, not the page, so a later distinct one still gets attempts.
+
+ Deliberately ignores ``src``: the common widgets carry a per-render cache-buster there, so
+ keying on it would mint a fresh identity on every re-render and never reach the cap.
+ """
+ marks = []
+ for control in interactive_challenge_controls(evidence.get("challenge_controls")):
+ mark = control.get("data_sitekey") or control.get("selector") or control.get("id")
+ if mark:
+ marks.append(str(mark))
+ return "|".join(sorted(marks)) or str(evidence.get("current_url") or "")
+
+
+def _record_challenge_encounter(
+ ctx: AgentContext,
+ carrier: ChallengeEvidenceSource,
+ *,
+ outcome: str,
+) -> None:
+ """Record on the interaction that met the challenge, whether or not it was passed.
+
+ Synthesis derives boundaries from ``composition_challenge_carrier`` over the recorded
+ interaction itself, so the carrier has to ride on a tool_name it already emits for; a
+ standalone solve_captcha interaction has no emitter and would be dropped.
+ """
+ trajectory = list(ctx.scout_trajectory)
+ if not trajectory:
+ return
+ entry = trajectory[-1]
+ challenge_state = dict(entry.get("challenge_state") or {})
+ challenge_state[CHALLENGE_EVIDENCE_SOURCE_KEY] = carrier.value
+ challenge_state["outcome"] = outcome
+ stamped = dict(entry)
+ stamped["challenge_state"] = challenge_state
+ trajectory[-1] = cast(ScoutedInteraction, stamped)
+ ctx.scout_trajectory = trajectory
+
+
+async def solve_challenge_when_evidence_settles(
+ ctx: AgentContext,
+ evidence: dict[str, Any] | None,
+ *,
+ url: str | None,
+ observed_after_interaction: bool,
+) -> bool:
+ """Try the shared solve routes on a challenge in just-settled evidence; True if it passed.
+
+ Call from a capture exit, never a tool hook: a settled packet describes the page as it now
+ is, and ``observed_after_interaction`` follows from the seam — the act-observe capture runs
+ after a recorded interaction, an inspection capture has none to attribute.
+ """
+ if not evidence:
+ return False
+ carrier = composition_challenge_carrier(evidence)
+ if carrier is None:
+ return False
+
+ # Record on detection rather than on success: an unsolved challenge is a fact the draft
+ # carries forward, and exploration continues either way. Nothing below terminates the turn.
+ if observed_after_interaction:
+ _record_challenge_encounter(ctx, carrier, outcome="detected")
+
+ # Cost gate, deliberately after the record above: a challenge that cannot be solved must stop
+ # being paid for, but must never stop being reported to synthesis.
+ # Counted before the attempt, so a transient failure spends one too — deliberate, because the
+ # alternative lets a route that keeps erroring retry without bound.
+ identity = _challenge_identity(evidence)
+ attempts = ctx.challenge_solve_attempts.get(identity, 0)
+ if attempts >= _MAX_CHALLENGE_SOLVE_ATTEMPTS:
+ return False
+ ctx.challenge_solve_attempts[identity] = attempts + 1
+
+ # Read once so the handlers below report a failure without touching the context again.
+ organization_id = ctx.organization_id
+ try:
+ # Resolving the browser is inside the guard because it can raise, and a challenge the
+ # scout merely failed to solve must never be what ends exploration.
+ browser_state = await resolve_browser_state_for_context(ctx)
+ if browser_state is None:
+ return False
+ page = await browser_state.get_working_page()
+ if page is None:
+ return False
+ # In-DOM checkbox, then the Turnstile extension, then the reCAPTCHA token route.
+ await _code_block_solve_captcha_builtin(page, organization_id=organization_id)
+ except CodeBlockCaptchaError:
+ LOG.info(
+ "copilot_scout_captcha_solve_failed",
+ url=url,
+ organization_id=organization_id,
+ )
+ if observed_after_interaction:
+ _record_challenge_encounter(ctx, carrier, outcome="unsolved")
+ return False
+ except Exception:
+ LOG.warning(
+ "copilot_scout_captcha_solve_exception",
+ url=url,
+ organization_id=organization_id,
+ exc_info=True,
+ )
+ return False
+
+ # "attempted", not "solved": the routes return normally when their probes match nothing, so a
+ # carrier asserted by challenge_state alone can complete here without anything being solved.
+ if observed_after_interaction:
+ _record_challenge_encounter(ctx, carrier, outcome="attempted")
+ LOG.info(
+ "copilot_scout_captcha_solved",
+ url=url,
+ organization_id=organization_id,
+ )
+ return True
+
+
def _page_evidence_has_selector(value: Any, selector: str) -> bool:
if isinstance(value, dict):
if value.get("selector") == selector:
@@ -1000,7 +1128,9 @@ def _mint_current_loaded_result_source(
return loaded_results
-async def _scout_act_observe_page_evidence(ctx: AgentContext, *, url: str) -> dict[str, Any] | None:
+async def _scout_act_observe_page_evidence(
+ ctx: AgentContext, *, url: str, observed_after_interaction: bool = False
+) -> dict[str, Any] | None:
"""Run the bounded page-side extractor right after a scout interaction.
Degrades to None on timeout or error so the interaction result is never
@@ -1022,8 +1152,9 @@ async def _scout_act_observe_page_evidence(ctx: AgentContext, *, url: str) -> di
outcome = "error"
else:
outcome = _scout_act_observe_capture_outcome(parsed, started=started, timeout_seconds=timeout_seconds)
- if outcome == "hollow" and parsed is not None:
- first_hollow = parsed
+ if parsed is not None and (outcome == "hollow" or challenge_evidence_unsettled(parsed)):
+ first_packet = parsed
+ first_outcome = outcome
remaining_seconds = timeout_seconds - (time.monotonic() - started)
if remaining_seconds <= 0:
ctx.last_scout_act_observe_recapture_result = "not_attempted_no_budget"
@@ -1040,15 +1171,15 @@ async def _scout_act_observe_page_evidence(ctx: AgentContext, *, url: str) -> di
ctx, inspected_url=url, current_url=url, timeout_seconds=remaining_seconds
)
except Exception:
- parsed = first_hollow
- outcome = "hollow"
+ parsed = first_packet
+ outcome = first_outcome
ctx.last_scout_act_observe_recapture_result = (
"timeout" if time.monotonic() - started >= timeout_seconds else "error"
)
else:
if recaptured is None:
- parsed = first_hollow
- outcome = "hollow"
+ parsed = first_packet
+ outcome = first_outcome
ctx.last_scout_act_observe_recapture_result = _scout_act_observe_no_payload_result(
started=started, timeout_seconds=timeout_seconds
)
@@ -1056,8 +1187,17 @@ async def _scout_act_observe_page_evidence(ctx: AgentContext, *, url: str) -> di
recaptured_outcome = _scout_act_observe_capture_outcome(
recaptured, started=started, timeout_seconds=timeout_seconds
)
- parsed = recaptured
- outcome = recaptured_outcome
+ # Never trade down: a page that navigated mid-recapture would otherwise
+ # lose the form the first capture proved, or erase the challenge signal
+ # that justified re-looking while still reporting a bounded schema.
+ if (
+ first_outcome == "attached" and recaptured_outcome != "attached"
+ ) or challenge_signal_regressed(first_packet, recaptured):
+ parsed = first_packet
+ outcome = first_outcome
+ else:
+ parsed = recaptured
+ outcome = recaptured_outcome
ctx.last_scout_act_observe_recapture_result = recaptured_outcome
ctx.last_scout_act_observe_outcome = outcome
ctx.last_scout_act_observe_packet = parsed
@@ -1071,6 +1211,9 @@ async def _scout_act_observe_page_evidence(ctx: AgentContext, *, url: str) -> di
recapture_attempted=ctx.last_scout_act_observe_recapture_attempted,
recapture_result=ctx.last_scout_act_observe_recapture_result,
)
+ await solve_challenge_when_evidence_settles(
+ ctx, parsed, url=url, observed_after_interaction=observed_after_interaction
+ )
return parsed
@@ -1094,7 +1237,7 @@ async def _register_scout_interaction_observation(
evidence["interaction_source_url"] = source_url.strip()
page_evidence: dict[str, Any] | None = None
if tool_name in _ACT_OBSERVE_TOOLS:
- parsed = await _scout_act_observe_page_evidence(ctx, url=url)
+ parsed = await _scout_act_observe_page_evidence(ctx, url=url, observed_after_interaction=True)
# Admission (credit axis) is decoupled from the hollow outcome (no-progress axis): a page
# that rendered witnessed value content is bindable even when it exposes no actionable schema.
if parsed is not None and (has_bounded_page_schema(parsed) or has_witnessed_value_content(parsed)):
@@ -1577,6 +1720,10 @@ async def _auto_act_on_repeat(ctx: AgentContext, result: dict[str, Any], *, url:
role=role,
accessible_name=accessible_name,
)
+ # This path records its click after the capture, so the encounter is attributed here instead.
+ auto_act_carrier = composition_challenge_carrier(post_evidence) if post_evidence else None
+ if auto_act_carrier is not None:
+ _record_challenge_encounter(ctx, auto_act_carrier, outcome="detected")
for key in ("next_action", "next_action_reason", "actionable_targets"):
data.pop(key, None)
diff --git a/skyvern/forge/sdk/workflow/models/block.py b/skyvern/forge/sdk/workflow/models/block.py
index 82ec5c94f..1df999685 100644
--- a/skyvern/forge/sdk/workflow/models/block.py
+++ b/skyvern/forge/sdk/workflow/models/block.py
@@ -118,6 +118,7 @@ from skyvern.forge.sdk.models import Step, StepStatus
from skyvern.forge.sdk.schemas.files import FileInfo
from skyvern.forge.sdk.schemas.task_v2 import TaskV2Status
from skyvern.forge.sdk.schemas.tasks import Task, TaskOutput, TaskStatus
+from skyvern.forge.sdk.schemas.totp_codes import OTPType
from skyvern.forge.sdk.services import google_drive_service, google_oauth_service, sftp_service
from skyvern.forge.sdk.services.bitwarden import BitwardenConstants
from skyvern.forge.sdk.services.credentials import AzureVaultConstants, OnePasswordConstants, generate_totp_code
@@ -3697,38 +3698,34 @@ def _register_code_block_secret(workflow_run_context: WorkflowRunContext, value:
workflow_run_context.secrets[fresh_key] = value
-async def _resolve_code_block_otp(
- credential_parameter_key: str,
- organization_id: str | None,
- workflow_run_id: str | None,
- *,
- budget_seconds: int,
-) -> str:
- """Resolve a fresh OTP at call time for one credential: re-mint its TOTP (the staleness
- fix) or poll its email/SMS/magic-link, registering the value as a secret before return.
- The run context is re-resolved from workflow_run_id, never captured in the bound method's
- closure, so user code cannot reach the seed through the method's cells."""
- if not workflow_run_id:
- raise CodeBlockOTPError("OTP is unavailable: no workflow run is associated with this code block.")
+class OTPResult(str):
+ """A fetched OTP that still knows whether it is a code to type or a link to open.
- workflow_run_context = app.WORKFLOW_CONTEXT_MANAGER.get_workflow_run_context(workflow_run_id)
- if workflow_run_context is None:
- raise CodeBlockOTPError("OTP is unavailable: the workflow run context could not be resolved.")
+ Subclasses ``str`` so it can be typed into a field directly, while ``is_link`` lets
+ authored code branch without pattern-matching the value.
+ """
- otp_value = otp_service.try_generate_totp_for_credential(
- workflow_run_context, credential_parameter_key, workflow_run_id
- )
- if otp_value is not None:
- _register_code_block_secret(workflow_run_context, otp_value.value)
- return otp_value.value
+ otp_type: OTPType
- totp_identifier = workflow_run_context.get_credential_totp_identifier(credential_parameter_key)
- if not totp_identifier:
- raise CodeBlockOTPError(
- "No OTP source is configured for this credential. "
- "Add a TOTP secret or an email/SMS identifier to the credential."
- )
+ def __new__(cls, value: str, otp_type: OTPType) -> OTPResult:
+ result = super().__new__(cls, value)
+ result.otp_type = otp_type
+ return result
+ @property
+ def is_link(self) -> bool:
+ return self.otp_type is OTPType.MAGIC_LINK
+
+
+async def _poll_code_block_otp(
+ totp_identifier: str,
+ organization_id: str | None,
+ workflow_run_id: str,
+ workflow_run_context: WorkflowRunContext,
+ *,
+ budget_seconds: int,
+ subject: str,
+) -> otp_service.OTPValue:
if not organization_id:
raise CodeBlockOTPError("OTP is unavailable: no organization is associated with this code block.")
@@ -3752,15 +3749,85 @@ async def _resolve_code_block_otp(
except asyncio.TimeoutError:
raise CodeBlockOTPError(f"OTP was not received within {budget_seconds} seconds.")
except (NoTOTPVerificationCodeFound, FailedToGetTOTPVerificationCode):
- raise CodeBlockOTPError("OTP could not be retrieved for this credential.")
+ raise CodeBlockOTPError(f"OTP could not be retrieved for {subject}.")
if polled is None:
- raise CodeBlockOTPError("OTP could not be retrieved for this credential.")
+ raise CodeBlockOTPError(f"OTP could not be retrieved for {subject}.")
_register_code_block_secret(workflow_run_context, polled.value)
+ return polled
+
+
+def _code_block_workflow_run_context(workflow_run_id: str) -> WorkflowRunContext:
+ workflow_run_context = app.WORKFLOW_CONTEXT_MANAGER.get_workflow_run_context(workflow_run_id)
+ if workflow_run_context is None:
+ raise CodeBlockOTPError("OTP is unavailable: the workflow run context could not be resolved.")
+ return workflow_run_context
+
+
+async def _resolve_code_block_otp(
+ credential_parameter_key: str,
+ organization_id: str | None,
+ workflow_run_id: str | None,
+ *,
+ budget_seconds: int,
+) -> str:
+ """Resolve a fresh OTP at call time for one credential: re-mint its TOTP (the staleness
+ fix) or poll its email/SMS/magic-link, registering the value as a secret before return.
+ The run context is re-resolved from workflow_run_id, never captured in the bound method's
+ closure, so user code cannot reach the seed through the method's cells."""
+ if not workflow_run_id:
+ raise CodeBlockOTPError("OTP is unavailable: no workflow run is associated with this code block.")
+ workflow_run_context = _code_block_workflow_run_context(workflow_run_id)
+
+ otp_value = otp_service.try_generate_totp_for_credential(
+ workflow_run_context, credential_parameter_key, workflow_run_id
+ )
+ if otp_value is not None:
+ _register_code_block_secret(workflow_run_context, otp_value.value)
+ return otp_value.value
+
+ totp_identifier = workflow_run_context.get_credential_totp_identifier(credential_parameter_key)
+ if not totp_identifier:
+ raise CodeBlockOTPError(
+ "No OTP source is configured for this credential. "
+ "Add a TOTP secret or an email/SMS identifier to the credential."
+ )
+
+ polled = await _poll_code_block_otp(
+ totp_identifier,
+ organization_id,
+ workflow_run_id,
+ workflow_run_context,
+ budget_seconds=budget_seconds,
+ subject="this credential",
+ )
return polled.value
+async def _resolve_code_block_otp_for_identifier(
+ totp_identifier: str,
+ organization_id: str | None,
+ workflow_run_id: str | None,
+ *,
+ budget_seconds: int,
+) -> OTPResult:
+ """Resolve a fresh OTP for a bare email/SMS identifier, with no credential in play."""
+ if not workflow_run_id:
+ raise CodeBlockOTPError("OTP is unavailable: no workflow run is associated with this code block.")
+ workflow_run_context = _code_block_workflow_run_context(workflow_run_id)
+
+ polled = await _poll_code_block_otp(
+ totp_identifier,
+ organization_id,
+ workflow_run_id,
+ workflow_run_context,
+ budget_seconds=budget_seconds,
+ subject="this address",
+ )
+ return OTPResult(polled.value, polled.get_otp_type())
+
+
def _bind_code_block_otp(
credential_parameter_key: str,
organization_id: str | None,
@@ -3780,11 +3847,25 @@ def _bind_code_block_otp(
return otp
-async def _code_block_otp_builtin(credential: object) -> str:
- """Top-level ``await otp(credential)`` sugar that forwards to the credential's bound otp()."""
+async def _code_block_otp_builtin(
+ credential: object,
+ *,
+ organization_id: str | None = None,
+ workflow_run_id: str | None = None,
+) -> str:
+ """Top-level ``await otp(...)`` sugar. Given a credential it forwards to that credential's
+ bound otp(); given a bare email/SMS address it resolves against the address alone and
+ returns an OTPResult that still knows whether it is a code or a link."""
+ if isinstance(credential, str):
+ return await _resolve_code_block_otp_for_identifier(
+ credential,
+ organization_id,
+ workflow_run_id,
+ budget_seconds=settings.CODE_BLOCK_OTP_POLL_TIMEOUT_SECONDS,
+ )
bound = getattr(credential, "otp", None)
if not callable(bound):
- raise CodeBlockOTPError("otp() expects a credential with an OTP source.")
+ raise CodeBlockOTPError("otp() expects a credential with an OTP source, or an email address.")
return await bound()
@@ -3875,6 +3956,11 @@ class CodeBlock(Block):
organization_id=organization_id,
workflow_run_id=workflow_run_id,
)
+ safe_vars["otp"] = partial(
+ _code_block_otp_builtin,
+ organization_id=organization_id,
+ workflow_run_id=workflow_run_id,
+ )
parameter_defaults: dict[str, Any] = {}
if parameters:
for key, value in parameters.items():
diff --git a/skyvern/webeye/utils/page.py b/skyvern/webeye/utils/page.py
index 9a6c7757a..688ab290c 100644
--- a/skyvern/webeye/utils/page.py
+++ b/skyvern/webeye/utils/page.py
@@ -251,7 +251,7 @@ async def _current_viewpoint_screenshot_helper(
)
return screenshot
except TimeoutError as e:
- LOG.error(
+ LOG.warning(
"Screenshot timeout",
timeout_ms=timeout,
url=url,
diff --git a/tests/unit/test_code_block_sandbox.py b/tests/unit/test_code_block_sandbox.py
index 056fa5cc5..bd1eb8418 100644
--- a/tests/unit/test_code_block_sandbox.py
+++ b/tests/unit/test_code_block_sandbox.py
@@ -1457,6 +1457,124 @@ class TestCodeBlockOtpBuiltinDelegates:
assert called["bound"] is True
+class TestCodeBlockOtpForIdentifier:
+ """A bare address resolves an OTP with no credential in play, and keeps its code/link form."""
+
+ @staticmethod
+ def _patch_poll(monkeypatch: pytest.MonkeyPatch, wrc, polled):
+ from skyvern.forge.sdk.workflow.models import block as block_module
+
+ _patch_context_resolution(monkeypatch, wrc)
+
+ async def fake_get_workflow_run(*args: object, **kwargs: object) -> _FakeWorkflowRun:
+ return _FakeWorkflowRun()
+
+ captured: dict[str, object] = {}
+
+ async def fake_poll(**kwargs: object):
+ captured.update(kwargs)
+ return polled
+
+ monkeypatch.setattr(
+ block_module.app.DATABASE.workflow_runs, "get_workflow_run", fake_get_workflow_run, raising=False
+ )
+ monkeypatch.setattr(block_module.otp_service, "poll_otp_value", fake_poll)
+ return captured
+
+ @pytest.mark.asyncio
+ async def test_resolves_without_any_credential(self, monkeypatch: pytest.MonkeyPatch) -> None:
+ from unittest.mock import MagicMock
+
+ from skyvern.forge.sdk.workflow.context_manager import WorkflowRunContext
+ from skyvern.forge.sdk.workflow.models.block import _resolve_code_block_otp_for_identifier
+ from skyvern.services.otp_service import OTPValue
+
+ wrc = WorkflowRunContext(
+ workflow_title="t",
+ workflow_id="w",
+ workflow_permanent_id="wpid",
+ workflow_run_id=_WORKFLOW_RUN_ID,
+ aws_client=MagicMock(),
+ )
+ assert wrc.values == {}
+ assert wrc.credential_totp_identifiers == {}
+
+ captured = self._patch_poll(monkeypatch, wrc, OTPValue(value="778899"))
+
+ result = await _resolve_code_block_otp_for_identifier(
+ "[email protected]", _ORG_ID, _WORKFLOW_RUN_ID, budget_seconds=120
+ )
+
+ assert result == "778899"
+ assert captured["totp_identifier"] == "[email protected]"
+ assert captured["organization_id"] == _ORG_ID
+ # Same run-start anchoring as the credential-bound path.
+ assert captured["created_after"] == _FakeWorkflowRun().started_at
+ # Same secret registration as the credential-bound path.
+ assert "778899" in wrc.secrets.values()
+
+ @pytest.mark.asyncio
+ async def test_magic_link_is_flagged_as_link(self, monkeypatch: pytest.MonkeyPatch) -> None:
+ from skyvern.forge.sdk.workflow.models.block import _resolve_code_block_otp_for_identifier
+ from skyvern.services.otp_service import OTPValue
+
+ wrc = _build_wrc_with_identifier()
+ self._patch_poll(monkeypatch, wrc, OTPValue(value="https://example.com/magic?t=abc"))
+
+ result = await _resolve_code_block_otp_for_identifier(
+ "[email protected]", _ORG_ID, _WORKFLOW_RUN_ID, budget_seconds=120
+ )
+
+ assert result.is_link is True
+ assert result == "https://example.com/magic?t=abc"
+
+ @pytest.mark.asyncio
+ async def test_code_is_not_flagged_as_link(self, monkeypatch: pytest.MonkeyPatch) -> None:
+ from skyvern.forge.sdk.workflow.models.block import _resolve_code_block_otp_for_identifier
+ from skyvern.services.otp_service import OTPValue
+
+ wrc = _build_wrc_with_identifier()
+ self._patch_poll(monkeypatch, wrc, OTPValue(value="445566"))
+
+ result = await _resolve_code_block_otp_for_identifier(
+ "[email protected]", _ORG_ID, _WORKFLOW_RUN_ID, budget_seconds=120
+ )
+
+ assert result.is_link is False
+ # Usable directly as the value to type, without unwrapping.
+ assert result == "445566"
+ assert result.strip() == "445566"
+
+ @pytest.mark.asyncio
+ async def test_builtin_routes_a_string_to_the_identifier_path(self, monkeypatch: pytest.MonkeyPatch) -> None:
+ from skyvern.forge.sdk.workflow.models.block import CodeBlock as _CB
+ from skyvern.services.otp_service import OTPValue
+
+ wrc = _build_wrc_with_identifier()
+ captured = self._patch_poll(monkeypatch, wrc, OTPValue(value="112233"))
+
+ otp_builtin = _CB.build_safe_vars()["otp"]
+ result = await otp_builtin("[email protected]", organization_id=_ORG_ID, workflow_run_id=_WORKFLOW_RUN_ID)
+
+ assert result == "112233"
+ assert captured["totp_identifier"] == "[email protected]"
+
+ @pytest.mark.asyncio
+ async def test_credential_path_still_returns_a_plain_string(self, monkeypatch: pytest.MonkeyPatch) -> None:
+ """otp(credential) is unchanged: a bare str, with no code/link attribute grafted on."""
+ from skyvern.forge.sdk.workflow.models.block import OTPResult, _resolve_code_block_otp
+ from skyvern.services.otp_service import OTPValue
+
+ wrc = _build_wrc_with_identifier()
+ self._patch_poll(monkeypatch, wrc, OTPValue(value="https://example.com/magic?t=xyz"))
+
+ code = await _resolve_code_block_otp(_CREDENTIAL_KEY, _ORG_ID, _WORKFLOW_RUN_ID, budget_seconds=120)
+
+ assert type(code) is str
+ assert not isinstance(code, OTPResult)
+ assert code == "https://example.com/magic?t=xyz"
+
+
class TestCodeBlockOtpNoLeak:
"""AC3: the resolved OTP is masked in the persisted output and never leaks the seed."""
diff --git a/tests/unit/test_copilot_code_block_synthesis.py b/tests/unit/test_copilot_code_block_synthesis.py
index b3d4b917f..1acdd8b5d 100644
--- a/tests/unit/test_copilot_code_block_synthesis.py
+++ b/tests/unit/test_copilot_code_block_synthesis.py
@@ -5076,3 +5076,34 @@ class TestScoutedSpineOmissionDigest:
def test_digest_is_cross_process_stable_sha256_not_salted_hash(self) -> None:
digest = _scouted_spine_omission_digest([self._record("#search-submit", 0)])
assert digest == "8065b147a155c4e35cab8b3b35da9beab958cddc9c355b6200bac957878954ec"
+
+
+def _captcha_click(challenge_state: dict[str, Any] | None = None) -> dict[str, Any]:
+ click: dict[str, Any] = {
+ "tool_name": "click",
+ "selector": "button.btn--login",
+ "source_url": "https://example.com/login",
+ "trajectory_index": 0,
+ }
+ if challenge_state is not None:
+ click["challenge_state"] = challenge_state
+ return click
+
+
+def test_stamped_challenge_carrier_emits_solve_captcha_after_the_click() -> None:
+ result = synthesize_code_block(
+ [_captcha_click({"evidence_source": "challenge_state"})],
+ strict_selectors=True,
+ )
+ assert result is not None
+ lines = [line.strip() for line in result.code.splitlines() if line.strip()]
+ click_index = next(i for i, line in enumerate(lines) if ".click()" in line)
+ solve_index = next(i for i, line in enumerate(lines) if line == "await solve_captcha(page)")
+ assert solve_index > click_index
+ assert lines[click_index + 1 : solve_index] == ['await page.wait_for_load_state("domcontentloaded")']
+
+
+def test_unstamped_click_emits_no_solve_captcha() -> None:
+ result = synthesize_code_block([_captcha_click()], strict_selectors=True)
+ assert result is not None
+ assert "solve_captcha" not in result.code
diff --git a/tests/unit/test_copilot_completion_verification.py b/tests/unit/test_copilot_completion_verification.py
index 0973c3e64..cb9fc2a01 100644
--- a/tests/unit/test_copilot_completion_verification.py
+++ b/tests/unit/test_copilot_completion_verification.py
@@ -6074,8 +6074,10 @@ async def test_active_run_terminal_evidence_sample_matches_current_page(
inspected_url: str,
current_url: str,
active_run_terminal_sample: bool = False,
+ solve_challenges: bool = True,
) -> tuple[dict, None]:
captured["active_run_terminal_sample"] = active_run_terminal_sample
+ captured["solve_challenges"] = solve_challenges
return (
{
"inspected_url": inspected_url,
@@ -6117,6 +6119,8 @@ async def test_active_run_terminal_evidence_sample_matches_current_page(
assert sample.current_url == "https://example.com/cart"
assert sample.page_evidence["observed_during_active_workflow_run"] is True
assert captured["active_run_terminal_sample"] is True
+ # The executor still owns this page; sampling it must never drive a challenge solve.
+ assert captured["solve_challenges"] is False
assert "PART-001-TEST" in str(captured["prompt"])
diff --git a/tests/unit/test_copilot_composition_scrape.py b/tests/unit/test_copilot_composition_scrape.py
index 4250e023f..cccfe1513 100644
--- a/tests/unit/test_copilot_composition_scrape.py
+++ b/tests/unit/test_copilot_composition_scrape.py
@@ -12,6 +12,7 @@ from unittest.mock import AsyncMock
import pytest
from skyvern.forge.sdk.copilot import tools
+from skyvern.forge.sdk.copilot.composition_evidence import parse_composition_structured
from skyvern.forge.sdk.copilot.tools import _normalized_inspect_url, _same_inspect_target
@@ -84,3 +85,144 @@ async def test_recapture_skips_raw_get_html_after_cap_drop(monkeypatch: pytest.M
# First iteration's raw read is cap-dropped; the settle retry skips it entirely.
assert raw_calls["n"] == 1
settle_sleep.assert_awaited_once_with(tools.composition_capture._COMPOSITION_HOLLOW_RECAPTURE_DELAY_SECONDS)
+
+
+def _challenge_signalled_structured_payload(*, with_form: bool = True) -> dict:
+ """Anti-bot token in the title only, no rendered challenge control: signalled, no carrier."""
+ payload: dict = {
+ "page_title": "Just a moment...",
+ "anti_bot_indicators": ["just a moment"],
+ "challenge_controls": [],
+ "body_has_markup": True,
+ "forms": [],
+ }
+ if with_form:
+ payload["forms"] = [
+ {
+ "fields": [{"name": "email", "label": "Email", "type": "text", "selector": "#email"}],
+ "submit_controls": [{"text": "Log in", "type": "submit", "selector": "#go"}],
+ }
+ ]
+ return payload
+
+
+async def test_unrendered_challenge_keeps_structured_packet_when_relooks_run_out(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ """get_html reads body only, so re-parsing there drops a title-derived challenge signal.
+ Exhausting the re-looks must keep the structured packet instead of trading down to it."""
+ packet = parse_composition_structured(
+ _challenge_signalled_structured_payload(),
+ inspected_url="https://example.com/login",
+ current_url="https://example.com/login",
+ )
+
+ async def fake_structured(ctx: object, **_kwargs: object) -> dict:
+ return dict(packet)
+
+ async def identity(ctx: object, evidence: dict) -> dict:
+ return evidence
+
+ # Body-only read: the anti-bot token lives in <title>, so it is absent here by construction.
+ get_html = AsyncMock(return_value=(_BOUNDED_HTML, None, False, False))
+ monkeypatch.setattr(tools.composition_capture, "_composition_get_structured_evidence", fake_structured)
+ monkeypatch.setattr(tools.composition_capture, "_composition_get_html", get_html)
+ monkeypatch.setattr(
+ tools.composition_capture, "_augment_composition_evidence_with_computed_obstruction_candidates", identity
+ )
+ monkeypatch.setattr(tools.composition_capture, "_augment_composition_evidence_with_visual_fallback", identity)
+ settle_sleep = AsyncMock()
+ monkeypatch.setattr(tools.composition_capture, "asyncio", _AsyncioSleepProxy(settle_sleep))
+
+ evidence, html_error = await tools._capture_composition_evidence(
+ SimpleNamespace(), inspected_url="https://example.com/login", current_url="https://example.com/login"
+ )
+
+ assert html_error is None
+ assert evidence is not None
+ assert evidence["challenge_state"]["detected"] is True
+ assert evidence["challenge_state"]["indicators"] == ["just a moment"]
+ # The body-only re-read must not happen at all; it is what erased the signal.
+ get_html.assert_not_awaited()
+ assert settle_sleep.await_count == tools.composition_capture._COMPOSITION_HOLLOW_RECAPTURE_RETRIES
+
+
+async def test_settled_structured_packet_pays_no_extra_relook(monkeypatch: pytest.MonkeyPatch) -> None:
+ """A bounded page with no challenge signal is already settled: no settle, no second capture."""
+ packet = parse_composition_structured(
+ {
+ "page_title": "Results",
+ "body_has_markup": True,
+ "forms": [
+ {
+ "fields": [{"name": "q", "label": "Query", "type": "text", "selector": "#q"}],
+ "submit_controls": [{"text": "Go", "type": "submit", "selector": "#go"}],
+ }
+ ],
+ },
+ inspected_url="https://example.com/s",
+ current_url="https://example.com/s",
+ )
+ calls = {"n": 0}
+
+ async def fake_structured(ctx: object, **_kwargs: object) -> dict:
+ calls["n"] += 1
+ return dict(packet)
+
+ async def identity(ctx: object, evidence: dict) -> dict:
+ return evidence
+
+ monkeypatch.setattr(tools.composition_capture, "_composition_get_structured_evidence", fake_structured)
+ monkeypatch.setattr(
+ tools.composition_capture, "_augment_composition_evidence_with_computed_obstruction_candidates", identity
+ )
+ settle_sleep = AsyncMock()
+ monkeypatch.setattr(tools.composition_capture, "asyncio", _AsyncioSleepProxy(settle_sleep))
+
+ evidence, html_error = await tools._capture_composition_evidence(
+ SimpleNamespace(), inspected_url="https://example.com/s", current_url="https://example.com/s"
+ )
+
+ assert html_error is None
+ assert tools.has_bounded_page_schema(evidence)
+ assert calls["n"] == 1
+ settle_sleep.assert_not_awaited()
+
+
+async def test_signalled_packet_survives_extractor_blinking_mid_loop(monkeypatch: pytest.MonkeyPatch) -> None:
+ """A later attempt whose extractor returns None must not clobber the retained packet with a
+ body-only reparse: an interstitial that reloads while we re-look fails skyvern_evaluate."""
+ packet = parse_composition_structured(
+ _challenge_signalled_structured_payload(),
+ inspected_url="https://example.com/login",
+ current_url="https://example.com/login",
+ )
+ payloads = iter([dict(packet), None, None])
+
+ async def fake_structured(ctx: object, **_kwargs: object) -> dict | None:
+ return next(payloads)
+
+ async def identity(ctx: object, evidence: dict) -> dict:
+ return evidence
+
+ get_html = AsyncMock(return_value=(_BOUNDED_HTML, None, False, False))
+ monkeypatch.setattr(tools.composition_capture, "_composition_get_structured_evidence", fake_structured)
+ monkeypatch.setattr(tools.composition_capture, "_composition_get_html", get_html)
+ monkeypatch.setattr(
+ tools.composition_capture, "_augment_composition_evidence_with_computed_obstruction_candidates", identity
+ )
+ monkeypatch.setattr(tools.composition_capture, "_augment_composition_evidence_with_visual_fallback", identity)
+ monkeypatch.setattr(tools.composition_capture, "asyncio", _AsyncioSleepProxy(AsyncMock()))
+
+ evidence, html_error = await tools._capture_composition_evidence(
+ SimpleNamespace(), inspected_url="https://example.com/login", current_url="https://example.com/login"
+ )
+
+ assert html_error is None
+ assert evidence is not None
+ assert evidence["challenge_state"]["detected"] is True
+ assert evidence["challenge_state"]["indicators"] == ["just a moment"]
+ get_html.assert_not_awaited()
diff --git a/tests/unit/test_copilot_diagnosis_repair_contract.py b/tests/unit/test_copilot_diagnosis_repair_contract.py
index 74369e281..c9d9b4fd2 100644
--- a/tests/unit/test_copilot_diagnosis_repair_contract.py
+++ b/tests/unit/test_copilot_diagnosis_repair_contract.py
@@ -2538,7 +2538,7 @@ async def test_bounded_seam_capture_is_stored_stamped_without_touching_budget(
captured.pop("workflow_run_id", None)
async def fake_capture(
- _ctx: CopilotContext, *, inspected_url: str, current_url: str
+ _ctx: CopilotContext, *, inspected_url: str, current_url: str, solve_challenges: bool = True
) -> tuple[dict[str, object], None]:
return dict(captured), None
@@ -2588,7 +2588,9 @@ async def test_failed_seam_capture_neutralizes_non_matching_evidence(
ctx.block_authoring_policy = BlockAuthoringPolicy.CODE_ONLY_BROWSER
ctx.composition_page_evidence = stale
- async def fake_capture(_ctx: CopilotContext, *, inspected_url: str, current_url: str) -> tuple[None, None]:
+ async def fake_capture(
+ _ctx: CopilotContext, *, inspected_url: str, current_url: str, solve_challenges: bool = True
+ ) -> tuple[None, None]:
return None, None
monkeypatch.setattr(run_execution_module, "_capture_composition_evidence", fake_capture)
@@ -2610,7 +2612,9 @@ async def test_failed_seam_capture_preserves_clean_matching_evidence(
clean["workflow_run_id"] = "wr_failed"
ctx.composition_page_evidence = clean
- async def fake_capture(_ctx: CopilotContext, *, inspected_url: str, current_url: str) -> tuple[None, None]:
+ async def fake_capture(
+ _ctx: CopilotContext, *, inspected_url: str, current_url: str, solve_challenges: bool = True
+ ) -> tuple[None, None]:
return None, None
monkeypatch.setattr(run_execution_module, "_capture_composition_evidence", fake_capture)
diff --git a/tests/unit/test_copilot_scout_act_observe.py b/tests/unit/test_copilot_scout_act_observe.py
index 20a653567..134e306d8 100644
--- a/tests/unit/test_copilot_scout_act_observe.py
+++ b/tests/unit/test_copilot_scout_act_observe.py
@@ -17,6 +17,10 @@ from structlog.testing import capture_logs
from skyvern.config import settings
from skyvern.forge.sdk.copilot import tools as tools_module
+from skyvern.forge.sdk.copilot.challenge_evidence import (
+ ChallengeEvidenceSource,
+ composition_challenge_carrier,
+)
from skyvern.forge.sdk.copilot.composition_evidence import (
_auto_credit_interaction_observation,
has_bounded_page_schema,
@@ -89,6 +93,19 @@ def _bounded_extractor_payload() -> dict[str, Any]:
}
+def _bounded_challenge_signalled_payload(challenge_controls: list[dict[str, Any]] | None = None) -> dict[str, Any]:
+ """Login page whose schema is already bounded while the challenge widget is only
+ signalled by a keyword; ``challenge_controls`` supplies the rendered carrier."""
+ payload = _bounded_extractor_payload()
+ payload["anti_bot_indicators"] = ["captcha"]
+ payload["challenge_controls"] = challenge_controls or []
+ return payload
+
+
+def _rendered_iframe_challenge_control() -> dict[str, Any]:
+ return {"tag": "iframe", "selector": 'iframe[title="reCAPTCHA"]', "text": ""}
+
+
def _kv_only_extractor_payload() -> dict[str, Any]:
return {
"page_title": "Provider Record",
@@ -123,6 +140,8 @@ def _ctx(*, server: Any = None, source_url: str | None = _SOURCE_URL) -> SimpleN
discovery_mcp_server=server,
scouted_interactions=[],
scout_trajectory=[],
+ challenge_solve_attempts={},
+ organization_id="o_test",
prior_fill_carry=[],
fill_carry_rebound_done=False,
pending_scout_source_url=source_url,
@@ -912,6 +931,99 @@ class TestActObserveRecaptureSettle:
assert sleeps == []
assert ctx.last_scout_act_observe_outcome == "attached"
+ @pytest.mark.asyncio
+ async def test_attached_challenge_without_carrier_settles_and_recaptures(
+ self, monkeypatch: pytest.MonkeyPatch
+ ) -> None:
+ monkeypatch.setattr(settings, "COPILOT_CLICK_SETTLE_DELAY_SECONDS", 0.6)
+ sleeps: list[float] = []
+
+ async def record_sleep(seconds: float) -> None:
+ sleeps.append(seconds)
+
+ monkeypatch.setattr(scouting_module.asyncio, "sleep", record_sleep)
+ ctx = _ctx(
+ server=_server_returning_sequence(
+ [
+ _bounded_challenge_signalled_payload(),
+ _bounded_challenge_signalled_payload([_rendered_iframe_challenge_control()]),
+ ]
+ )
+ )
+
+ packet = await _scout_act_observe_page_evidence(ctx, url=_LANDING_URL)
+
+ assert sleeps == [0.6]
+ assert ctx.last_scout_act_observe_recapture_attempted is True
+ assert ctx.last_scout_act_observe_outcome == "attached"
+ assert composition_challenge_carrier(packet) is ChallengeEvidenceSource.CHALLENGE_STATE
+
+ @pytest.mark.asyncio
+ async def test_attached_keyword_only_challenge_recapture_still_yields_no_carrier(
+ self, monkeypatch: pytest.MonkeyPatch
+ ) -> None:
+ monkeypatch.setattr(settings, "COPILOT_CLICK_SETTLE_DELAY_SECONDS", 0.6)
+
+ async def record_sleep(_seconds: float) -> None:
+ return None
+
+ monkeypatch.setattr(scouting_module.asyncio, "sleep", record_sleep)
+ ctx = _ctx(
+ server=_server_returning_sequence(
+ [
+ _bounded_challenge_signalled_payload(),
+ _bounded_challenge_signalled_payload(),
+ ]
+ )
+ )
+
+ packet = await _scout_act_observe_page_evidence(ctx, url=_LANDING_URL)
+
+ assert ctx.last_scout_act_observe_recapture_attempted is True
+ assert composition_challenge_carrier(packet) is None
+
+ @pytest.mark.asyncio
+ async def test_recapture_may_not_erase_the_challenge_signal(self, monkeypatch: pytest.MonkeyPatch) -> None:
+ """A recapture that still reports a bounded schema but has lost the challenge signal
+ must not replace the signalled packet: the signal is the visual fallback's only trigger."""
+ monkeypatch.setattr(settings, "COPILOT_CLICK_SETTLE_DELAY_SECONDS", 0.6)
+
+ async def record_sleep(_seconds: float) -> None:
+ return None
+
+ monkeypatch.setattr(scouting_module.asyncio, "sleep", record_sleep)
+ unsignalled = _bounded_extractor_payload()
+ ctx = _ctx(server=_server_returning_sequence([_bounded_challenge_signalled_payload(), unsignalled]))
+
+ packet = await _scout_act_observe_page_evidence(ctx, url=_LANDING_URL)
+
+ assert packet is not None
+ assert packet["challenge_state"]["detected"] is True
+ assert packet["challenge_state"]["indicators"] == ["captcha"]
+
+ @pytest.mark.asyncio
+ async def test_attached_packet_survives_a_hollow_recapture(self, monkeypatch: pytest.MonkeyPatch) -> None:
+ monkeypatch.setattr(settings, "COPILOT_CLICK_SETTLE_DELAY_SECONDS", 0.6)
+
+ async def record_sleep(_seconds: float) -> None:
+ return None
+
+ monkeypatch.setattr(scouting_module.asyncio, "sleep", record_sleep)
+ ctx = _ctx(
+ server=_server_returning_sequence(
+ [
+ _bounded_challenge_signalled_payload(),
+ {"page_title": "Results", "forms": []},
+ ]
+ )
+ )
+
+ packet = await _scout_act_observe_page_evidence(ctx, url=_LANDING_URL)
+
+ assert ctx.last_scout_act_observe_outcome == "attached"
+ assert packet is not None
+ assert packet["forms"]
+
@pytest.mark.asyncio
async def test_capture_log_carries_container_and_relation_counts(self) -> None:
ctx = _ctx(server=_server_returning(_kv_only_extractor_payload()))
diff --git a/tests/unit/test_copilot_scout_captcha_solve.py b/tests/unit/test_copilot_scout_captcha_solve.py
new file mode 100644
index 000000000..4f28e7cf0
--- /dev/null
+++ b/tests/unit/test_copilot_scout_captcha_solve.py
@@ -0,0 +1,485 @@
+"""Scout CAPTCHA solving (SKY-12967): on settled evidence the scout tries the shared
+solve routes, records the encounter, and continues whether or not it passed."""
+
+from __future__ import annotations
+
+from types import SimpleNamespace
+from typing import Any
+from unittest.mock import AsyncMock, patch
+
+import pytest
+
+from skyvern.forge.sdk.copilot.challenge_evidence import (
+ ChallengeEvidenceSource,
+ composition_challenge_carrier,
+)
+from skyvern.forge.sdk.copilot.runtime import AgentContext
+from skyvern.forge.sdk.copilot.tools.scouting import (
+ _challenge_identity,
+ solve_challenge_when_evidence_settles,
+)
+from skyvern.forge.sdk.workflow.models.block import CodeBlockCaptchaError
+
+
+def _mock_page_evidence_with_challenge() -> dict[str, Any]:
+ """Page evidence with a detected challenge carrier."""
+ return {
+ "current_url": "https://example.com/login",
+ "challenge_controls": [
+ {
+ "tag": "div",
+ "text": "Please verify you're human",
+ "id": "captcha-container",
+ }
+ ],
+ "challenge_state": {
+ "detected": True,
+ "evidence_source": "challenge_state",
+ },
+ }
+
+
+def _mock_page_evidence_no_challenge() -> dict[str, Any]:
+ """Page evidence with no challenge."""
+ return {
+ "current_url": "https://example.com/results",
+ "forms": [
+ {
+ "fields": [
+ {"name": "q", "type": "text", "selector": "#search"},
+ ],
+ }
+ ],
+ }
+
+
+def _mock_context() -> AgentContext:
+ """Minimal AgentContext stand-in, restricted to attributes the real class declares.
+
+ A SimpleNamespace accepts any attribute, so an implementation reading a field
+ AgentContext does not have still passes here unless the names are checked.
+ """
+ attrs = {
+ "organization_id": "org_123",
+ "workflow_id": "w_456",
+ "workflow_permanent_id": "wpid_456",
+ "scouted_interactions": [],
+ "scout_trajectory": [],
+ "challenge_solve_attempts": {},
+ "never_captured_obligation": None,
+ "last_evaluate_actionable_signature": None,
+ "last_scout_observation_trajectory_index": None,
+ }
+ declared = set(AgentContext.__annotations__)
+ undeclared = sorted(set(attrs) - declared)
+ assert not undeclared, f"not AgentContext fields: {undeclared}"
+ return SimpleNamespace(**attrs)
+
+
+async def test_attempt_scout_captcha_solve_no_challenge() -> None:
+ """When no challenge is detected, returns False without attempting solve."""
+ ctx = _mock_context()
+ page_evidence = _mock_page_evidence_no_challenge()
+
+ result = await solve_challenge_when_evidence_settles(
+ ctx, page_evidence, url="https://example.com/results", observed_after_interaction=True
+ )
+
+ assert result is False
+ assert len(ctx.scouted_interactions) == 0
+
+
+async def test_attempt_scout_captcha_solve_no_page_evidence() -> None:
+ """When page_evidence is None, returns False."""
+ ctx = _mock_context()
+
+ result = await solve_challenge_when_evidence_settles(
+ ctx, None, url="https://example.com/login", observed_after_interaction=True
+ )
+
+ assert result is False
+ assert len(ctx.scouted_interactions) == 0
+
+
+async def test_attempt_scout_captcha_solve_challenge_detected_and_solved() -> None:
+ """When a challenge is detected and successfully solved, calls solve and records."""
+ ctx = _mock_context()
+ page_evidence = _mock_page_evidence_with_challenge()
+
+ mock_page = AsyncMock()
+ mock_browser_state = AsyncMock()
+ mock_browser_state.get_working_page.return_value = mock_page
+
+ ctx.scout_trajectory = [
+ {"tool_name": "click", "selector": "button.btn--login", "source_url": "https://example.com/login"}
+ ]
+
+ with (
+ patch(
+ "skyvern.forge.sdk.copilot.tools.scouting.resolve_browser_state_for_context",
+ return_value=mock_browser_state,
+ ),
+ patch(
+ "skyvern.forge.sdk.copilot.tools.scouting._code_block_solve_captcha_builtin",
+ new_callable=AsyncMock,
+ ) as mock_solve,
+ ):
+ result = await solve_challenge_when_evidence_settles(
+ ctx, page_evidence, url="https://example.com/login", observed_after_interaction=True
+ )
+
+ assert result is True
+ mock_solve.assert_called_once()
+ boundary = ctx.scout_trajectory[-1]
+ assert boundary["tool_name"] == "click"
+ assert composition_challenge_carrier(boundary) is ChallengeEvidenceSource.CHALLENGE_STATE
+
+
+async def test_attempt_scout_captcha_solve_stamps_the_boundary_even_when_the_solve_fails() -> None:
+ """A failed solve still marks the encounter, so synthesis emits at that boundary."""
+ ctx = _mock_context()
+ page_evidence = _mock_page_evidence_with_challenge()
+ ctx.scout_trajectory = [
+ {"tool_name": "click", "selector": "button.btn--login", "source_url": "https://example.com/login"}
+ ]
+
+ mock_page = AsyncMock()
+ mock_browser_state = AsyncMock()
+ mock_browser_state.get_working_page.return_value = mock_page
+
+ with (
+ patch(
+ "skyvern.forge.sdk.copilot.tools.scouting.resolve_browser_state_for_context",
+ return_value=mock_browser_state,
+ ),
+ patch(
+ "skyvern.forge.sdk.copilot.tools.scouting._code_block_solve_captcha_builtin",
+ new_callable=AsyncMock,
+ side_effect=CodeBlockCaptchaError("CAPTCHA could not be solved"),
+ ),
+ ):
+ result = await solve_challenge_when_evidence_settles(
+ ctx, page_evidence, url="https://example.com/login", observed_after_interaction=True
+ )
+
+ assert result is False
+ assert composition_challenge_carrier(ctx.scout_trajectory[-1]) is ChallengeEvidenceSource.CHALLENGE_STATE
+
+
+async def test_attempt_scout_captcha_solve_no_browser_context() -> None:
+ """When browser_state is None, returns False."""
+ ctx = _mock_context()
+ page_evidence = _mock_page_evidence_with_challenge()
+
+ with patch(
+ "skyvern.forge.sdk.copilot.tools.scouting.resolve_browser_state_for_context",
+ return_value=None,
+ ):
+ result = await solve_challenge_when_evidence_settles(
+ ctx, page_evidence, url="https://example.com/login", observed_after_interaction=True
+ )
+
+ assert result is False
+ assert len(ctx.scouted_interactions) == 0
+
+
+async def test_attempt_scout_captcha_solve_no_page() -> None:
+ """When get_working_page returns None, returns False."""
+ ctx = _mock_context()
+ page_evidence = _mock_page_evidence_with_challenge()
+
+ mock_browser_state = AsyncMock()
+ mock_browser_state.get_working_page.return_value = None
+
+ with patch(
+ "skyvern.forge.sdk.copilot.tools.scouting.resolve_browser_state_for_context",
+ return_value=mock_browser_state,
+ ):
+ result = await solve_challenge_when_evidence_settles(
+ ctx, page_evidence, url="https://example.com/login", observed_after_interaction=True
+ )
+
+ assert result is False
+ assert len(ctx.scouted_interactions) == 0
+
+
+async def test_inspection_capture_solves_without_attributing_the_carrier() -> None:
+ """A capture with no interaction behind it still solves, but stamps nothing.
+
+ Attribution is a property of the seam: the act-observe capture follows a recorded
+ interaction, an inspection capture does not, so only the former can name a boundary.
+ """
+ ctx = _mock_context()
+ ctx.scout_trajectory = [
+ {"tool_name": "click", "selector": "button.btn--login", "source_url": "https://example.com/login"}
+ ]
+ mock_page = AsyncMock()
+ mock_browser_state = AsyncMock()
+ mock_browser_state.get_working_page.return_value = mock_page
+
+ with (
+ patch(
+ "skyvern.forge.sdk.copilot.tools.scouting.resolve_browser_state_for_context",
+ return_value=mock_browser_state,
+ ),
+ patch(
+ "skyvern.forge.sdk.copilot.tools.scouting._code_block_solve_captcha_builtin",
+ new_callable=AsyncMock,
+ ) as mock_solve,
+ ):
+ result = await solve_challenge_when_evidence_settles(
+ ctx,
+ _mock_page_evidence_with_challenge(),
+ url="https://example.com/login",
+ observed_after_interaction=False,
+ )
+
+ assert result is True
+ mock_solve.assert_called_once()
+ assert composition_challenge_carrier(ctx.scout_trajectory[-1]) is None
+
+
+async def test_unsolvable_challenge_is_recorded_and_exploration_continues() -> None:
+ """Every route failing records the encounter as unsolved and returns; it never terminates."""
+ ctx = _mock_context()
+ ctx.scout_trajectory = [
+ {"tool_name": "click", "selector": "button.btn--login", "source_url": "https://example.com/login"}
+ ]
+ mock_page = AsyncMock()
+ mock_browser_state = AsyncMock()
+ mock_browser_state.get_working_page.return_value = mock_page
+
+ with (
+ patch(
+ "skyvern.forge.sdk.copilot.tools.scouting.resolve_browser_state_for_context",
+ return_value=mock_browser_state,
+ ),
+ patch(
+ "skyvern.forge.sdk.copilot.tools.scouting._code_block_solve_captcha_builtin",
+ new_callable=AsyncMock,
+ side_effect=CodeBlockCaptchaError("CAPTCHA could not be solved"),
+ ),
+ ):
+ result = await solve_challenge_when_evidence_settles(
+ ctx,
+ _mock_page_evidence_with_challenge(),
+ url="https://example.com/login",
+ observed_after_interaction=True,
+ )
+
+ assert result is False
+ boundary = ctx.scout_trajectory[-1]
+ assert composition_challenge_carrier(boundary) is ChallengeEvidenceSource.CHALLENGE_STATE
+ assert boundary["challenge_state"]["outcome"] == "unsolved"
+
+
+async def test_encounter_is_recorded_even_when_no_page_is_available_to_solve_on() -> None:
+ """Recording happens on detection, so an unreachable page cannot lose the fact."""
+ ctx = _mock_context()
+ ctx.scout_trajectory = [
+ {"tool_name": "click", "selector": "button.btn--login", "source_url": "https://example.com/login"}
+ ]
+ mock_browser_state = AsyncMock()
+ mock_browser_state.get_working_page.return_value = None
+
+ with patch(
+ "skyvern.forge.sdk.copilot.tools.scouting.resolve_browser_state_for_context",
+ return_value=mock_browser_state,
+ ):
+ result = await solve_challenge_when_evidence_settles(
+ ctx,
+ _mock_page_evidence_with_challenge(),
+ url="https://example.com/login",
+ observed_after_interaction=True,
+ )
+
+ assert result is False
+ assert composition_challenge_carrier(ctx.scout_trajectory[-1]) is ChallengeEvidenceSource.CHALLENGE_STATE
+
+
+async def test_a_raising_browser_lookup_cannot_end_exploration() -> None:
+ """Resolving the browser can raise; that must not propagate out and stop the turn."""
+ ctx = _mock_context()
+ ctx.scout_trajectory = [
+ {"tool_name": "click", "selector": "button.btn--login", "source_url": "https://example.com/login"}
+ ]
+
+ with patch(
+ "skyvern.forge.sdk.copilot.tools.scouting.resolve_browser_state_for_context",
+ side_effect=RuntimeError("browser session gone"),
+ ):
+ result = await solve_challenge_when_evidence_settles(
+ ctx,
+ _mock_page_evidence_with_challenge(),
+ url="https://example.com/login",
+ observed_after_interaction=True,
+ )
+
+ assert result is False
+ assert composition_challenge_carrier(ctx.scout_trajectory[-1]) is ChallengeEvidenceSource.CHALLENGE_STATE
+
+
+async def test_a_raising_page_lookup_cannot_end_exploration() -> None:
+ """Same for the working-page lookup, which is a separate await that can raise."""
+ ctx = _mock_context()
+ mock_browser_state = AsyncMock()
+ mock_browser_state.get_working_page.side_effect = RuntimeError("target closed")
+
+ with patch(
+ "skyvern.forge.sdk.copilot.tools.scouting.resolve_browser_state_for_context",
+ return_value=mock_browser_state,
+ ):
+ result = await solve_challenge_when_evidence_settles(
+ ctx,
+ _mock_page_evidence_with_challenge(),
+ url="https://example.com/login",
+ observed_after_interaction=False,
+ )
+
+ assert result is False
+
+
+async def test_a_stuck_challenge_stops_being_paid_for_but_keeps_being_recorded() -> None:
+ """The cap bounds paid attempts; the boundary record is never gated by it."""
+ ctx = _mock_context()
+ mock_browser_state = AsyncMock()
+ mock_browser_state.get_working_page.return_value = AsyncMock()
+
+ with (
+ patch(
+ "skyvern.forge.sdk.copilot.tools.scouting.resolve_browser_state_for_context",
+ return_value=mock_browser_state,
+ ),
+ patch(
+ "skyvern.forge.sdk.copilot.tools.scouting._code_block_solve_captcha_builtin",
+ new_callable=AsyncMock,
+ side_effect=CodeBlockCaptchaError("CAPTCHA could not be solved"),
+ ) as mock_solve,
+ ):
+ for _ in range(6):
+ ctx.scout_trajectory = [{"tool_name": "click", "selector": "#s", "source_url": "u"}]
+ await solve_challenge_when_evidence_settles(
+ ctx,
+ _mock_page_evidence_with_challenge(),
+ url="https://example.com/login",
+ observed_after_interaction=True,
+ )
+ # Recorded on every pass, including the ones the cap refused to pay for.
+ assert composition_challenge_carrier(ctx.scout_trajectory[-1]) is ChallengeEvidenceSource.CHALLENGE_STATE
+
+ assert mock_solve.await_count == 3
+
+
+async def test_a_distinct_challenge_gets_its_own_attempts() -> None:
+ """Keying on the challenge, not the page, keeps a later distinct one from inheriting the cap."""
+ ctx = _mock_context()
+ # Key off what production actually computes; a hand-picked key can never collide and
+ # would make this pass whether or not per-challenge keying works.
+ spent = dict(_mock_page_evidence_with_challenge())
+ spent["challenge_controls"] = [{"tag": "div", "id": "c", "data_sitekey": "sitekey-already-spent"}]
+ ctx.challenge_solve_attempts = {_challenge_identity(spent): 99}
+ mock_browser_state = AsyncMock()
+ mock_browser_state.get_working_page.return_value = AsyncMock()
+
+ with (
+ patch(
+ "skyvern.forge.sdk.copilot.tools.scouting.resolve_browser_state_for_context",
+ return_value=mock_browser_state,
+ ),
+ patch(
+ "skyvern.forge.sdk.copilot.tools.scouting._code_block_solve_captcha_builtin",
+ new_callable=AsyncMock,
+ ) as mock_solve,
+ ):
+ result = await solve_challenge_when_evidence_settles(
+ ctx,
+ _mock_page_evidence_with_challenge(),
+ url="https://example.com/login",
+ observed_after_interaction=False,
+ )
+
+ assert result is True
+ mock_solve.assert_awaited_once()
+
+
+async def test_a_rotating_widget_src_cannot_slip_the_attempt_cap() -> None:
+ """Common widgets re-render with a fresh cache-buster in src; identity must not follow it."""
+ ctx = _mock_context()
+ mock_browser_state = AsyncMock()
+ mock_browser_state.get_working_page.return_value = AsyncMock()
+
+ def _evidence(nonce: str) -> dict[str, Any]:
+ packet = dict(_mock_page_evidence_with_challenge())
+ packet["challenge_controls"] = [
+ {
+ "tag": "iframe",
+ "selector": "#captcha-frame",
+ "src": f"https://example.com/recaptcha/api2/anchor?k=sitekey&cb={nonce}",
+ }
+ ]
+ return packet
+
+ assert _challenge_identity(_evidence("aaa")) == _challenge_identity(_evidence("zzz"))
+
+ with (
+ patch(
+ "skyvern.forge.sdk.copilot.tools.scouting.resolve_browser_state_for_context",
+ return_value=mock_browser_state,
+ ),
+ patch(
+ "skyvern.forge.sdk.copilot.tools.scouting._code_block_solve_captcha_builtin",
+ new_callable=AsyncMock,
+ side_effect=CodeBlockCaptchaError("CAPTCHA could not be solved"),
+ ) as mock_solve,
+ ):
+ for nonce in ("a", "b", "c", "d", "e", "f"):
+ await solve_challenge_when_evidence_settles(
+ ctx, _evidence(nonce), url="https://example.com/login", observed_after_interaction=False
+ )
+
+ assert mock_solve.await_count == 3
+
+
+async def test_a_state_only_carrier_is_not_reported_as_solved() -> None:
+ """The routes return normally when their probes match nothing, so completing is not solving."""
+ ctx = _mock_context()
+ ctx.scout_trajectory = [{"tool_name": "click", "selector": "#s", "source_url": "u"}]
+ evidence = {
+ "current_url": "https://example.com/login",
+ # No challenge_controls: the carrier is asserted by state alone, which the DOM probes
+ # inside the solve routes cannot match.
+ "challenge_state": {"evidence_source": "challenge_state", "requires_human_verification": True},
+ }
+ mock_browser_state = AsyncMock()
+ mock_browser_state.get_working_page.return_value = AsyncMock()
+
+ with (
+ patch(
+ "skyvern.forge.sdk.copilot.tools.scouting.resolve_browser_state_for_context",
+ return_value=mock_browser_state,
+ ),
+ patch(
+ "skyvern.forge.sdk.copilot.tools.scouting._code_block_solve_captcha_builtin",
+ new_callable=AsyncMock,
+ ),
+ ):
+ await solve_challenge_when_evidence_settles(
+ ctx, evidence, url="https://example.com/login", observed_after_interaction=True
+ )
+
+ assert ctx.scout_trajectory[-1]["challenge_state"]["outcome"] == "attempted"