Skip to content

Automate browser based workflows with AI

SKY-13061: reach reCAPTCHA v2 anchor inside its frame (#7787)

Andrew Neilson82fa4ff2f3b3
diff --git a/skyvern/forge/sdk/workflow/models/block.py b/skyvern/forge/sdk/workflow/models/block.py
index b395e1515..6316637bb 100644
--- a/skyvern/forge/sdk/workflow/models/block.py
+++ b/skyvern/forge/sdk/workflow/models/block.py
@@ -43,7 +43,7 @@ from jsonschema import Draft202012Validator
from jsonschema.exceptions import ValidationError
from opentelemetry import trace as otel_trace
from playwright.async_api import Error as PlaywrightError
-from playwright.async_api import Page
+from playwright.async_api import Frame, Page
from pydantic import BaseModel, Field, PrivateAttr, model_validator
from sqlalchemy.exc import InterfaceError, OperationalError
@@ -3645,6 +3645,12 @@ _CODE_BLOCK_RECAPTCHA_MARKER_SELECTOR = ", ".join(
'iframe[title*="recaptcha" i]',
)
)
+_CODE_BLOCK_RECAPTCHA_RESPONSE_SELECTOR = 'textarea[name="g-recaptcha-response"], textarea[id^="g-recaptcha-response"]'
+_CODE_BLOCK_RECAPTCHA_ANCHOR_HOSTS = ("www.google.com", "www.recaptcha.net")
+_CODE_BLOCK_RECAPTCHA_ANCHOR_PATHS = ("/recaptcha/api2/anchor", "/recaptcha/enterprise/anchor")
+_CODE_BLOCK_RECAPTCHA_ANCHOR_ARM_TIMEOUT_SECONDS = 5
+# Google's widget flips aria-checked after its own animation; a shorter wait reads as unsolved.
+_CODE_BLOCK_RECAPTCHA_ANCHOR_SETTLE_MS = 2_000
_CODE_BLOCK_CAPTCHA_CONTINUE_SELECTOR = ", ".join(
(
"[data-challenge-state] button[type='submit']",
@@ -3664,6 +3670,34 @@ async def _bounded_code_block_locator_count(locator: Any) -> int:
return 0
+def _is_trusted_code_block_recaptcha_anchor_url(frame_url: str | None) -> bool:
+ if not frame_url:
+ return False
+ try:
+ parsed = urlparse(frame_url)
+ except ValueError:
+ return False
+ hostname = (parsed.hostname or "").lower()
+ return (
+ parsed.scheme == "https"
+ and hostname in _CODE_BLOCK_RECAPTCHA_ANCHOR_HOSTS
+ and parsed.path in _CODE_BLOCK_RECAPTCHA_ANCHOR_PATHS
+ )
+
+
+async def _bounded_code_block_recaptcha_token_populated(scope: Frame | Page | RecordingPage) -> bool | None:
+ try:
+ async with asyncio.timeout(1):
+ fields = scope.locator(_CODE_BLOCK_RECAPTCHA_RESPONSE_SELECTOR)
+ for index in range(await fields.count()):
+ value = await fields.nth(index).input_value()
+ if value and value.strip().lower() not in {"undefined", "null"}:
+ return True
+ except (PlaywrightError, TimeoutError):
+ return None
+ return False
+
+
async def _code_block_solve_captcha_builtin(
page: Page | RecordingPage,
*,
@@ -3704,6 +3738,45 @@ async def _code_block_solve_captcha_builtin(
except Exception:
LOG.info("code block CAPTCHA checkbox arm did not solve", arm="dom_checkbox")
+ # A page-level locator cannot cross into reCAPTCHA's anchor iframe. Click the checkbox in-frame.
+ try:
+ async with asyncio.timeout(_CODE_BLOCK_RECAPTCHA_ANCHOR_ARM_TIMEOUT_SECONDS):
+ for frame in page.frames:
+ if not _is_trusted_code_block_recaptcha_anchor_url(frame.url):
+ continue
+ anchor = frame.locator("#recaptcha-anchor")
+ if await _bounded_code_block_locator_count(anchor) != 1:
+ continue
+ candidate = await anchor.first.element_handle()
+ if candidate is None or not (await candidate.is_visible()):
+ continue
+ # The handle is bound to the validated document. If the frame navigates after this
+ # re-check, Playwright detaches the handle instead of clicking the replacement page.
+ if not _is_trusted_code_block_recaptcha_anchor_url(frame.url):
+ continue
+ token_scope = frame.parent_frame or page
+ token_was_populated = await _bounded_code_block_recaptcha_token_populated(token_scope)
+ if await candidate.get_attribute("aria-checked") == "true":
+ break
+ page_url_before_click = urlparse(page.url)._replace(fragment="").geturl()
+ await candidate.click()
+ LOG.info("code block CAPTCHA anchor frame clicked", arm="recaptcha_anchor_frame")
+ await page.wait_for_timeout(_CODE_BLOCK_RECAPTCHA_ANCHOR_SETTLE_MS)
+ if frame.is_detached() and urlparse(page.url)._replace(fragment="").geturl() != page_url_before_click:
+ LOG.info("code block CAPTCHA anchor frame solved after navigation", arm="recaptcha_anchor_frame")
+ return
+ token_is_populated = await _bounded_code_block_recaptcha_token_populated(token_scope)
+ if (
+ await candidate.get_attribute("aria-checked") == "true"
+ and token_was_populated is False
+ and token_is_populated is True
+ ):
+ LOG.info("code block CAPTCHA anchor frame solved", arm="recaptcha_anchor_frame")
+ return
+ break
+ except Exception:
+ LOG.info("code block CAPTCHA anchor frame arm did not solve", arm="recaptcha_anchor_frame")
+
try:
if await app.AGENT_FUNCTION.auto_solve_captchas(page):
return
diff --git a/tests/unit/test_code_block_captcha.py b/tests/unit/test_code_block_captcha.py
index 8dd71f02e..e90abbe0d 100644
--- a/tests/unit/test_code_block_captcha.py
+++ b/tests/unit/test_code_block_captcha.py
@@ -1,17 +1,21 @@
from __future__ import annotations
+import asyncio
from unittest.mock import AsyncMock
import pytest
from skyvern.forge import app
+from skyvern.forge.sdk.workflow.models import block as block_module
from skyvern.forge.sdk.workflow.models.block import CodeBlock, CodeBlockCaptchaError
class FakeLocator:
- def __init__(self, *, count: int = 0, checked: bool = False) -> None:
+ def __init__(self, *, count: int = 0, checked: bool = False, input_values: list[str] | None = None) -> None:
self._count = count
self._checked = checked
+ self._input_values = list(input_values or [])
+ self._input_value_calls = 0
self.click = AsyncMock(side_effect=self._click)
async def _click(self) -> None:
@@ -29,23 +33,75 @@ class FakeLocator:
async def is_checked(self) -> bool:
return self._checked
+ async def get_attribute(self, name: str) -> str | None:
+ assert name == "aria-checked"
+ return "true" if self._checked else "false"
+
+ async def element_handle(self) -> FakeLocator:
+ return self
+
+ def nth(self, _index: int) -> FakeLocator:
+ return self
+
+ async def input_value(self) -> str:
+ if not self._input_values:
+ return ""
+ index = min(self._input_value_calls, len(self._input_values) - 1)
+ self._input_value_calls += 1
+ return self._input_values[index]
+
@property
def first(self) -> FakeLocator:
return self
+class FakeFrame:
+ def __init__(
+ self,
+ *,
+ url: str,
+ anchor: FakeLocator,
+ parent_frame: FakePage | None = None,
+ detached: bool = False,
+ ) -> None:
+ self.url = url
+ self.anchor = anchor
+ self.parent_frame = parent_frame
+ self.detached = detached
+
+ def locator(self, selector: str) -> FakeLocator:
+ assert selector == "#recaptcha-anchor"
+ return self.anchor
+
+ def is_detached(self) -> bool:
+ return self.detached
+
+
class FakePage:
- def __init__(self, *, checkbox: bool = False, recaptcha: bool = False) -> None:
+ def __init__(
+ self,
+ *,
+ checkbox: bool = False,
+ recaptcha: bool = False,
+ token_values: list[str] | None = None,
+ frames: list[FakeFrame] | None = None,
+ url: str = "https://app.example/login",
+ ) -> None:
self.checkbox = FakeLocator(count=1 if checkbox else 0)
self.challenge = FakeLocator(count=1 if recaptcha else 0)
+ self.recaptcha_token = FakeLocator(count=1 if token_values is not None else 0, input_values=token_values)
self.continue_button = FakeLocator(count=1 if checkbox else 0)
self.continue_button.click = AsyncMock(side_effect=self._continue)
+ self.frames = list(frames or [])
+ self.url = url
async def _continue(self) -> None:
self.checkbox._count = 0
self.challenge._count = 0
def locator(self, selector: str) -> FakeLocator:
+ if "g-recaptcha-response" in selector:
+ return self.recaptcha_token
if "checkbox" in selector:
return self.checkbox
if "button" in selector:
@@ -53,7 +109,7 @@ class FakePage:
return self.challenge
async def wait_for_timeout(self, _milliseconds: int) -> None:
- return None
+ await asyncio.sleep(0)
@pytest.mark.asyncio
@@ -101,6 +157,332 @@ async def test_real_sandbox_solve_captcha_clicks_unique_structural_checkbox(
agent_function.auto_solve_captchas.assert_not_awaited()
+ "anchor_url",
+ (
+ "https://www.google.com/recaptcha/api2/anchor",
+ "https://www.google.com/recaptcha/enterprise/anchor",
+ ),
+)
+async def test_real_sandbox_solve_captcha_clicks_recaptcha_anchor_in_frame(
+ monkeypatch: pytest.MonkeyPatch,
+ anchor_url: str,
+) -> None:
+ agent_function = type(
+ "AgentFunctionStub",
+ (),
+ {
+ "auto_solve_captchas": AsyncMock(return_value=False),
+ "solve_recaptcha_token": AsyncMock(return_value=False),
+ },
+ )()
+ monkeypatch.setattr(app, "AGENT_FUNCTION", agent_function)
+ anchor = FakeLocator(count=1)
+ parent_frame = FakePage(token_values=["", "opaque-token"])
+ page = FakePage(
+ recaptcha=True,
+ frames=[FakeFrame(url=anchor_url, anchor=anchor, parent_frame=parent_frame)],
+ )
+ block = CodeBlock.model_construct(code="await solve_captcha(page)", label="captcha_anchor")
+
+ await block.generate_async_user_function(block.code, page)()
+
+ anchor.click.assert_awaited_once_with()
+ assert await anchor.is_checked() is True
+ assert parent_frame.recaptcha_token._input_value_calls == 2
+ agent_function.auto_solve_captchas.assert_not_awaited()
+ agent_function.solve_recaptcha_token.assert_not_awaited()
+
+
+ "anchor_url",
+ (
+ "https://evil.example/recaptcha/api2/anchor",
+ "https://attacker.storage.googleapis.com/recaptcha/api2/anchor",
+ ),
+)
+async def test_real_sandbox_solve_captcha_does_not_click_untrusted_anchor_host(
+ monkeypatch: pytest.MonkeyPatch,
+ anchor_url: str,
+) -> None:
+ agent_function = type(
+ "AgentFunctionStub",
+ (),
+ {
+ "auto_solve_captchas": AsyncMock(return_value=True),
+ "solve_recaptcha_token": AsyncMock(return_value=False),
+ },
+ )()
+ monkeypatch.setattr(app, "AGENT_FUNCTION", agent_function)
+ anchor = FakeLocator(count=1)
+ page = FakePage(
+ recaptcha=True,
+ frames=[FakeFrame(url=anchor_url, anchor=anchor)],
+ )
+ block = CodeBlock.model_construct(code="await solve_captcha(page)", label="captcha_untrusted_anchor")
+
+ await block.generate_async_user_function(block.code, page)()
+
+ anchor.click.assert_not_awaited()
+ agent_function.auto_solve_captchas.assert_awaited_once_with(page)
+
+
+async def test_real_sandbox_solve_captcha_does_not_click_after_anchor_frame_navigates(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ agent_function = type(
+ "AgentFunctionStub",
+ (),
+ {
+ "auto_solve_captchas": AsyncMock(return_value=True),
+ "solve_recaptcha_token": AsyncMock(return_value=False),
+ },
+ )()
+ monkeypatch.setattr(app, "AGENT_FUNCTION", agent_function)
+ anchor = FakeLocator(count=1)
+ frame = FakeFrame(url="https://www.google.com/recaptcha/api2/anchor", anchor=anchor)
+
+ async def navigate_before_handle_returns() -> FakeLocator:
+ frame.url = "https://evil.example/recaptcha/api2/anchor"
+ return anchor
+
+ anchor.element_handle = AsyncMock(side_effect=navigate_before_handle_returns)
+ page = FakePage(recaptcha=True, frames=[frame])
+ block = CodeBlock.model_construct(code="await solve_captcha(page)", label="captcha_navigated_anchor")
+
+ await block.generate_async_user_function(block.code, page)()
+
+ anchor.click.assert_not_awaited()
+ agent_function.auto_solve_captchas.assert_awaited_once_with(page)
+
+
+ ("destination_url", "navigation_solved"),
+ (
+ ("https://app.example/account", True),
+ ("https://app.example/login#done", False),
+ ),
+)
+async def test_real_sandbox_solve_captcha_returns_after_callback_navigation(
+ monkeypatch: pytest.MonkeyPatch,
+ destination_url: str,
+ navigation_solved: bool,
+) -> None:
+ agent_function = type(
+ "AgentFunctionStub",
+ (),
+ {
+ "auto_solve_captchas": AsyncMock(return_value=not navigation_solved),
+ "solve_recaptcha_token": AsyncMock(return_value=False),
+ },
+ )()
+ monkeypatch.setattr(app, "AGENT_FUNCTION", agent_function)
+ anchor = FakeLocator(count=1)
+ parent_frame = FakePage(token_values=[""])
+ frame = FakeFrame(
+ url="https://www.google.com/recaptcha/api2/anchor",
+ anchor=anchor,
+ parent_frame=parent_frame,
+ )
+ page = FakePage(recaptcha=True, frames=[frame])
+
+ async def navigate_after_click(_milliseconds: int) -> None:
+ frame.detached = True
+ page.url = destination_url
+
+ page.wait_for_timeout = AsyncMock(side_effect=navigate_after_click)
+ block = CodeBlock.model_construct(code="await solve_captcha(page)", label="captcha_callback_navigation")
+
+ await block.generate_async_user_function(block.code, page)()
+
+ anchor.click.assert_awaited_once_with()
+ if navigation_solved:
+ agent_function.auto_solve_captchas.assert_not_awaited()
+ else:
+ agent_function.auto_solve_captchas.assert_awaited_once_with(page)
+ agent_function.solve_recaptcha_token.assert_not_awaited()
+
+
+async def test_real_sandbox_solve_captcha_rejects_prechecked_anchor_without_fresh_token(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ agent_function = type(
+ "AgentFunctionStub",
+ (),
+ {
+ "auto_solve_captchas": AsyncMock(return_value=False),
+ "solve_recaptcha_token": AsyncMock(return_value=True),
+ },
+ )()
+ monkeypatch.setattr(app, "AGENT_FUNCTION", agent_function)
+ anchor = FakeLocator(count=1, checked=True)
+ page = FakePage(
+ recaptcha=True,
+ token_values=["opaque-token"],
+ frames=[FakeFrame(url="https://www.google.com/recaptcha/api2/anchor", anchor=anchor)],
+ )
+ block = CodeBlock.model_construct(code="await solve_captcha(page)", label="captcha_anchor_prechecked")
+
+ await block.generate_async_user_function(block.code, page)()
+
+ anchor.click.assert_not_awaited()
+ agent_function.auto_solve_captchas.assert_awaited_once_with(page)
+ agent_function.solve_recaptcha_token.assert_awaited_once()
+
+
+async def test_real_sandbox_solve_captcha_falls_through_when_anchor_stays_unchecked(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ agent_function = type(
+ "AgentFunctionStub",
+ (),
+ {
+ "auto_solve_captchas": AsyncMock(return_value=True),
+ "solve_recaptcha_token": AsyncMock(return_value=False),
+ },
+ )()
+ monkeypatch.setattr(app, "AGENT_FUNCTION", agent_function)
+ anchor = FakeLocator(count=1)
+ anchor.click = AsyncMock()
+ page = FakePage(
+ recaptcha=True,
+ frames=[FakeFrame(url="https://www.google.com/recaptcha/api2/anchor", anchor=anchor)],
+ )
+ block = CodeBlock.model_construct(code="await solve_captcha(page)", label="captcha_anchor_unchecked")
+
+ await block.generate_async_user_function(block.code, page)()
+
+ anchor.click.assert_awaited_once_with()
+ agent_function.auto_solve_captchas.assert_awaited_once_with(page)
+ agent_function.solve_recaptcha_token.assert_not_awaited()
+
+
+async def test_real_sandbox_solve_captcha_falls_through_when_token_lags_checked_anchor(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ agent_function = type(
+ "AgentFunctionStub",
+ (),
+ {
+ "auto_solve_captchas": AsyncMock(return_value=False),
+ "solve_recaptcha_token": AsyncMock(return_value=True),
+ },
+ )()
+ monkeypatch.setattr(app, "AGENT_FUNCTION", agent_function)
+ anchor = FakeLocator(count=1)
+ page = FakePage(
+ recaptcha=True,
+ token_values=["", ""],
+ frames=[FakeFrame(url="https://www.google.com/recaptcha/api2/anchor", anchor=anchor)],
+ )
+ block = CodeBlock.model_construct(code="await solve_captcha(page)", label="captcha_anchor_late_token")
+
+ await block.generate_async_user_function(block.code, page)()
+
+ assert await anchor.is_checked() is True
+ agent_function.auto_solve_captchas.assert_awaited_once_with(page)
+ agent_function.solve_recaptcha_token.assert_awaited_once()
+
+
+async def test_real_sandbox_solve_captcha_rejects_preexisting_token(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ agent_function = type(
+ "AgentFunctionStub",
+ (),
+ {
+ "auto_solve_captchas": AsyncMock(return_value=False),
+ "solve_recaptcha_token": AsyncMock(return_value=True),
+ },
+ )()
+ monkeypatch.setattr(app, "AGENT_FUNCTION", agent_function)
+ anchor = FakeLocator(count=1)
+ page = FakePage(
+ recaptcha=True,
+ token_values=["stale-token"],
+ frames=[FakeFrame(url="https://www.google.com/recaptcha/api2/anchor", anchor=anchor)],
+ )
+ block = CodeBlock.model_construct(code="await solve_captcha(page)", label="captcha_anchor_stale_token")
+
+ await block.generate_async_user_function(block.code, page)()
+
+ anchor.click.assert_awaited_once_with()
+ agent_function.auto_solve_captchas.assert_awaited_once_with(page)
+ agent_function.solve_recaptcha_token.assert_awaited_once()
+
+
+async def test_real_sandbox_solve_captcha_rejects_inconclusive_token_baseline(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ agent_function = type(
+ "AgentFunctionStub",
+ (),
+ {
+ "auto_solve_captchas": AsyncMock(return_value=False),
+ "solve_recaptcha_token": AsyncMock(return_value=True),
+ },
+ )()
+ monkeypatch.setattr(app, "AGENT_FUNCTION", agent_function)
+ anchor = FakeLocator(count=1)
+ page = FakePage(
+ recaptcha=True,
+ token_values=["opaque-token"],
+ frames=[FakeFrame(url="https://www.google.com/recaptcha/api2/anchor", anchor=anchor)],
+ )
+ page.recaptcha_token.input_value = AsyncMock(
+ side_effect=[block_module.PlaywrightError("probe failed"), "opaque-token"]
+ )
+ block = CodeBlock.model_construct(code="await solve_captcha(page)", label="captcha_anchor_unknown_baseline")
+
+ await block.generate_async_user_function(block.code, page)()
+
+ anchor.click.assert_awaited_once_with()
+ agent_function.auto_solve_captchas.assert_awaited_once_with(page)
+ agent_function.solve_recaptcha_token.assert_awaited_once()
+
+
+async def test_real_sandbox_solve_captcha_bounds_unresponsive_anchor_arm(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ agent_function = type(
+ "AgentFunctionStub",
+ (),
+ {
+ "auto_solve_captchas": AsyncMock(return_value=True),
+ "solve_recaptcha_token": AsyncMock(return_value=False),
+ },
+ )()
+ monkeypatch.setattr(app, "AGENT_FUNCTION", agent_function)
+ monkeypatch.setattr(block_module, "_CODE_BLOCK_RECAPTCHA_ANCHOR_ARM_TIMEOUT_SECONDS", 0.01)
+
+ async def hang() -> None:
+ await asyncio.sleep(60)
+
+ anchor = FakeLocator(count=1)
+ anchor.click = AsyncMock(side_effect=hang)
+ page = FakePage(
+ recaptcha=True,
+ frames=[FakeFrame(url="https://www.google.com/recaptcha/api2/anchor", anchor=anchor)],
+ )
+ block = CodeBlock.model_construct(code="await solve_captcha(page)", label="captcha_anchor_timeout")
+
+ await asyncio.wait_for(block.generate_async_user_function(block.code, page)(), timeout=0.25)
+
+ anchor.click.assert_awaited_once_with()
+ agent_function.auto_solve_captchas.assert_awaited_once_with(page)
+ agent_function.solve_recaptcha_token.assert_not_awaited()
+
+
@pytest.mark.asyncio
async def test_real_sandbox_solve_captcha_raises_constant_sanitized_error(
monkeypatch: pytest.MonkeyPatch,