Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
Let copilot exploration solve login CAPTCHAs via the shared solve ladder (SKY-12967) (#7755)
Andrew Neilsonf3906e658ca4
diff --git a/skyvern/forge/sdk/copilot/runtime.py b/skyvern/forge/sdk/copilot/runtime.pyindex 00351da2c..9d5eef6ef 100644--- a/skyvern/forge/sdk/copilot/runtime.py+++ b/skyvern/forge/sdk/copilot/runtime.py@@ -6,7 +6,7 @@ import asyncioimport inspectfrom contextlib import asynccontextmanagerfrom dataclasses import dataclass, field-from typing import TYPE_CHECKING, Any, AsyncIterator, Awaitable, NotRequired, TypeAlias, TypedDict, cast+from typing import TYPE_CHECKING, Any, AsyncIterator, Awaitable, Literal, NotRequired, TypeAlias, TypedDict, castimport structlog@@ -239,6 +239,39 @@ class ScoutedInteraction(TypedDict):challenge_state: NotRequired[dict[str, Any]]+NeverCapturedObligationState: TypeAlias = Literal["armed", "captured", "consumed"]+++@dataclass(frozen=True)+class NeverCapturedReplayPayload:+ """Turn-ephemeral inputs required to retry the exact rejected authoring call."""++ params: dict[str, Any]+ allow_missing_credentials: bool | None = None+ allow_static_output_uncertainty: bool = False+ formation_prepared: bool = False+++@dataclass(frozen=True)+class NeverCapturedObligation:+ """Turn-ephemeral authority to re-scout one exact authored browser mutation."""++ identity_digest: str+ turn_id: str+ draft_fingerprint: str+ block_label: str+ site: str+ method: str+ normalized_receiver: str+ call_shape_digest: str+ expected_tool_name: str+ armed_after_trajectory_index: int+ expected_argument_literal: str | None = None+ captured_trajectory_index: int | None = None+ state: NeverCapturedObligationState = "armed"+ replay_payload: NeverCapturedReplayPayload | None = None++@dataclass(frozen=True)class PostRunPagePathInteractionWindow:structural_key: str@@ -430,6 +463,10 @@ class AgentContext:# Solve attempts already spent per challenge. A challenge that never passes is precisely the# one re-observed on every later capture, so cost otherwise grows with how stuck the turn is.challenge_solve_attempts: dict[str, int] = field(default_factory=dict)+ # One exact `never_captured` mutation may reopen scouting within this turn. The obligation is+ # completed only by a later generator-emitted canonical interaction, never by selector text alone.+ never_captured_obligation: NeverCapturedObligation | None = None+ never_captured_obligation_identity_history: set[str] = field(default_factory=set)# Latest typed reached-download target from the scout steer; the synthesizer compiles the terminal# expect_download step from it. Selector is the observed download link, not necessarily a trajectory click.reached_download_target: ReachedDownloadTarget | None = Nonediff --git a/tests/unit/test_copilot_scout_captcha_solve.py b/tests/unit/test_copilot_scout_captcha_solve.pyindex a048e07c0..4f28e7cf0 100644--- a/tests/unit/test_copilot_scout_captcha_solve.py+++ b/tests/unit/test_copilot_scout_captcha_solve.py@@ -66,6 +66,7 @@ def _mock_context() -> AgentContext:"scouted_interactions": [],"scout_trajectory": [],"challenge_solve_attempts": {},+ "never_captured_obligation": None,"last_evaluate_actionable_signature": None,"last_scout_observation_trajectory_index": None,}