Skip to content

Automate browser based workflows with AI

Let copilot exploration solve login CAPTCHAs via the shared solve ladder (SKY-12967) (#7755)

Andrew Neilsonf3906e658ca4
diff --git a/skyvern/forge/sdk/copilot/runtime.py b/skyvern/forge/sdk/copilot/runtime.py
index 00351da2c..9d5eef6ef 100644
--- a/skyvern/forge/sdk/copilot/runtime.py
+++ b/skyvern/forge/sdk/copilot/runtime.py
@@ -6,7 +6,7 @@ import asyncio
import inspect
from contextlib import asynccontextmanager
from dataclasses import dataclass, field
-from typing import TYPE_CHECKING, Any, AsyncIterator, Awaitable, NotRequired, TypeAlias, TypedDict, cast
+from typing import TYPE_CHECKING, Any, AsyncIterator, Awaitable, Literal, NotRequired, TypeAlias, TypedDict, cast
import structlog
@@ -239,6 +239,39 @@ class ScoutedInteraction(TypedDict):
challenge_state: NotRequired[dict[str, Any]]
+NeverCapturedObligationState: TypeAlias = Literal["armed", "captured", "consumed"]
+
+
+@dataclass(frozen=True)
+class NeverCapturedReplayPayload:
+ """Turn-ephemeral inputs required to retry the exact rejected authoring call."""
+
+ params: dict[str, Any]
+ allow_missing_credentials: bool | None = None
+ allow_static_output_uncertainty: bool = False
+ formation_prepared: bool = False
+
+
+@dataclass(frozen=True)
+class NeverCapturedObligation:
+ """Turn-ephemeral authority to re-scout one exact authored browser mutation."""
+
+ identity_digest: str
+ turn_id: str
+ draft_fingerprint: str
+ block_label: str
+ site: str
+ method: str
+ normalized_receiver: str
+ call_shape_digest: str
+ expected_tool_name: str
+ armed_after_trajectory_index: int
+ expected_argument_literal: str | None = None
+ captured_trajectory_index: int | None = None
+ state: NeverCapturedObligationState = "armed"
+ replay_payload: NeverCapturedReplayPayload | None = None
+
+
@dataclass(frozen=True)
class PostRunPagePathInteractionWindow:
structural_key: str
@@ -430,6 +463,10 @@ class AgentContext:
# Solve attempts already spent per challenge. A challenge that never passes is precisely the
# one re-observed on every later capture, so cost otherwise grows with how stuck the turn is.
challenge_solve_attempts: dict[str, int] = field(default_factory=dict)
+ # One exact `never_captured` mutation may reopen scouting within this turn. The obligation is
+ # completed only by a later generator-emitted canonical interaction, never by selector text alone.
+ never_captured_obligation: NeverCapturedObligation | None = None
+ never_captured_obligation_identity_history: set[str] = field(default_factory=set)
# Latest typed reached-download target from the scout steer; the synthesizer compiles the terminal
# expect_download step from it. Selector is the observed download link, not necessarily a trajectory click.
reached_download_target: ReachedDownloadTarget | None = None
diff --git a/tests/unit/test_copilot_scout_captcha_solve.py b/tests/unit/test_copilot_scout_captcha_solve.py
index a048e07c0..4f28e7cf0 100644
--- a/tests/unit/test_copilot_scout_captcha_solve.py
+++ b/tests/unit/test_copilot_scout_captcha_solve.py
@@ -66,6 +66,7 @@ def _mock_context() -> AgentContext:
"scouted_interactions": [],
"scout_trajectory": [],
"challenge_solve_attempts": {},
+ "never_captured_obligation": None,
"last_evaluate_actionable_signature": None,
"last_scout_observation_trajectory_index": None,
}