Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
SKY-13061: reach reCAPTCHA v2 anchor inside its frame (#7787)
Andrew Neilson82fa4ff2f3b3
diff --git a/skyvern/forge/sdk/workflow/models/block.py b/skyvern/forge/sdk/workflow/models/block.pyindex b395e1515..6316637bb 100644--- a/skyvern/forge/sdk/workflow/models/block.py+++ b/skyvern/forge/sdk/workflow/models/block.py@@ -43,7 +43,7 @@ from jsonschema import Draft202012Validatorfrom jsonschema.exceptions import ValidationErrorfrom opentelemetry import trace as otel_tracefrom playwright.async_api import Error as PlaywrightError-from playwright.async_api import Page+from playwright.async_api import Frame, Pagefrom pydantic import BaseModel, Field, PrivateAttr, model_validatorfrom sqlalchemy.exc import InterfaceError, OperationalError@@ -3645,6 +3645,12 @@ _CODE_BLOCK_RECAPTCHA_MARKER_SELECTOR = ", ".join('iframe[title*="recaptcha" i]',))+_CODE_BLOCK_RECAPTCHA_RESPONSE_SELECTOR = 'textarea[name="g-recaptcha-response"], textarea[id^="g-recaptcha-response"]'+_CODE_BLOCK_RECAPTCHA_ANCHOR_HOSTS = ("www.google.com", "www.recaptcha.net")+_CODE_BLOCK_RECAPTCHA_ANCHOR_PATHS = ("/recaptcha/api2/anchor", "/recaptcha/enterprise/anchor")+_CODE_BLOCK_RECAPTCHA_ANCHOR_ARM_TIMEOUT_SECONDS = 5+# Google's widget flips aria-checked after its own animation; a shorter wait reads as unsolved.+_CODE_BLOCK_RECAPTCHA_ANCHOR_SETTLE_MS = 2_000_CODE_BLOCK_CAPTCHA_CONTINUE_SELECTOR = ", ".join(("[data-challenge-state] button[type='submit']",@@ -3664,6 +3670,34 @@ async def _bounded_code_block_locator_count(locator: Any) -> int:return 0+def _is_trusted_code_block_recaptcha_anchor_url(frame_url: str | None) -> bool:+ if not frame_url:+ return False+ try:+ parsed = urlparse(frame_url)+ except ValueError:+ return False+ hostname = (parsed.hostname or "").lower()+ return (+ parsed.scheme == "https"+ and hostname in _CODE_BLOCK_RECAPTCHA_ANCHOR_HOSTS+ and parsed.path in _CODE_BLOCK_RECAPTCHA_ANCHOR_PATHS+ )+++async def _bounded_code_block_recaptcha_token_populated(scope: Frame | Page | RecordingPage) -> bool | None:+ try:+ async with asyncio.timeout(1):+ fields = scope.locator(_CODE_BLOCK_RECAPTCHA_RESPONSE_SELECTOR)+ for index in range(await fields.count()):+ value = await fields.nth(index).input_value()+ if value and value.strip().lower() not in {"undefined", "null"}:+ return True+ except (PlaywrightError, TimeoutError):+ return None+ return False++async def _code_block_solve_captcha_builtin(page: Page | RecordingPage,*,@@ -3704,6 +3738,45 @@ async def _code_block_solve_captcha_builtin(except Exception:LOG.info("code block CAPTCHA checkbox arm did not solve", arm="dom_checkbox")+ # A page-level locator cannot cross into reCAPTCHA's anchor iframe. Click the checkbox in-frame.+ try:+ async with asyncio.timeout(_CODE_BLOCK_RECAPTCHA_ANCHOR_ARM_TIMEOUT_SECONDS):+ for frame in page.frames:+ if not _is_trusted_code_block_recaptcha_anchor_url(frame.url):+ continue+ anchor = frame.locator("#recaptcha-anchor")+ if await _bounded_code_block_locator_count(anchor) != 1:+ continue+ candidate = await anchor.first.element_handle()+ if candidate is None or not (await candidate.is_visible()):+ continue+ # The handle is bound to the validated document. If the frame navigates after this+ # re-check, Playwright detaches the handle instead of clicking the replacement page.+ if not _is_trusted_code_block_recaptcha_anchor_url(frame.url):+ continue+ token_scope = frame.parent_frame or page+ token_was_populated = await _bounded_code_block_recaptcha_token_populated(token_scope)+ if await candidate.get_attribute("aria-checked") == "true":+ break+ page_url_before_click = urlparse(page.url)._replace(fragment="").geturl()+ await candidate.click()+ LOG.info("code block CAPTCHA anchor frame clicked", arm="recaptcha_anchor_frame")+ await page.wait_for_timeout(_CODE_BLOCK_RECAPTCHA_ANCHOR_SETTLE_MS)+ if frame.is_detached() and urlparse(page.url)._replace(fragment="").geturl() != page_url_before_click:+ LOG.info("code block CAPTCHA anchor frame solved after navigation", arm="recaptcha_anchor_frame")+ return+ token_is_populated = await _bounded_code_block_recaptcha_token_populated(token_scope)+ if (+ await candidate.get_attribute("aria-checked") == "true"+ and token_was_populated is False+ and token_is_populated is True+ ):+ LOG.info("code block CAPTCHA anchor frame solved", arm="recaptcha_anchor_frame")+ return+ break+ except Exception:+ LOG.info("code block CAPTCHA anchor frame arm did not solve", arm="recaptcha_anchor_frame")+try:if await app.AGENT_FUNCTION.auto_solve_captchas(page):returndiff --git a/tests/unit/test_code_block_captcha.py b/tests/unit/test_code_block_captcha.pyindex 8dd71f02e..e90abbe0d 100644--- a/tests/unit/test_code_block_captcha.py+++ b/tests/unit/test_code_block_captcha.py@@ -1,17 +1,21 @@from __future__ import annotations+import asynciofrom unittest.mock import AsyncMockimport pytestfrom skyvern.forge import app+from skyvern.forge.sdk.workflow.models import block as block_modulefrom skyvern.forge.sdk.workflow.models.block import CodeBlock, CodeBlockCaptchaErrorclass FakeLocator:- def __init__(self, *, count: int = 0, checked: bool = False) -> None:+ def __init__(self, *, count: int = 0, checked: bool = False, input_values: list[str] | None = None) -> None:self._count = countself._checked = checked+ self._input_values = list(input_values or [])+ self._input_value_calls = 0self.click = AsyncMock(side_effect=self._click)async def _click(self) -> None:@@ -29,23 +33,75 @@ class FakeLocator:async def is_checked(self) -> bool:return self._checked+ async def get_attribute(self, name: str) -> str | None:+ assert name == "aria-checked"+ return "true" if self._checked else "false"++ async def element_handle(self) -> FakeLocator:+ return self++ def nth(self, _index: int) -> FakeLocator:+ return self++ async def input_value(self) -> str:+ if not self._input_values:+ return ""+ index = min(self._input_value_calls, len(self._input_values) - 1)+ self._input_value_calls += 1+ return self._input_values[index]+@propertydef first(self) -> FakeLocator:return self+class FakeFrame:+ def __init__(+ self,+ *,+ url: str,+ anchor: FakeLocator,+ parent_frame: FakePage | None = None,+ detached: bool = False,+ ) -> None:+ self.url = url+ self.anchor = anchor+ self.parent_frame = parent_frame+ self.detached = detached++ def locator(self, selector: str) -> FakeLocator:+ assert selector == "#recaptcha-anchor"+ return self.anchor++ def is_detached(self) -> bool:+ return self.detached++class FakePage:- def __init__(self, *, checkbox: bool = False, recaptcha: bool = False) -> None:+ def __init__(+ self,+ *,+ checkbox: bool = False,+ recaptcha: bool = False,+ token_values: list[str] | None = None,+ frames: list[FakeFrame] | None = None,+ url: str = "https://app.example/login",+ ) -> None:self.checkbox = FakeLocator(count=1 if checkbox else 0)self.challenge = FakeLocator(count=1 if recaptcha else 0)+ self.recaptcha_token = FakeLocator(count=1 if token_values is not None else 0, input_values=token_values)self.continue_button = FakeLocator(count=1 if checkbox else 0)self.continue_button.click = AsyncMock(side_effect=self._continue)+ self.frames = list(frames or [])+ self.url = urlasync def _continue(self) -> None:self.checkbox._count = 0self.challenge._count = 0def locator(self, selector: str) -> FakeLocator:+ if "g-recaptcha-response" in selector:+ return self.recaptcha_tokenif "checkbox" in selector:return self.checkboxif "button" in selector:@@ -53,7 +109,7 @@ class FakePage:return self.challengeasync def wait_for_timeout(self, _milliseconds: int) -> None:- return None+ await asyncio.sleep(0)@pytest.mark.asyncio@@ -101,6 +157,332 @@ async def test_real_sandbox_solve_captcha_clicks_unique_structural_checkbox(agent_function.auto_solve_captchas.assert_not_awaited()+ "anchor_url",+ (+ "https://www.google.com/recaptcha/api2/anchor",+ "https://www.google.com/recaptcha/enterprise/anchor",+ ),+)+async def test_real_sandbox_solve_captcha_clicks_recaptcha_anchor_in_frame(+ monkeypatch: pytest.MonkeyPatch,+ anchor_url: str,+) -> None:+ agent_function = type(+ "AgentFunctionStub",+ (),+ {+ "auto_solve_captchas": AsyncMock(return_value=False),+ "solve_recaptcha_token": AsyncMock(return_value=False),+ },+ )()+ monkeypatch.setattr(app, "AGENT_FUNCTION", agent_function)+ anchor = FakeLocator(count=1)+ parent_frame = FakePage(token_values=["", "opaque-token"])+ page = FakePage(+ recaptcha=True,+ frames=[FakeFrame(url=anchor_url, anchor=anchor, parent_frame=parent_frame)],+ )+ block = CodeBlock.model_construct(code="await solve_captcha(page)", label="captcha_anchor")++ await block.generate_async_user_function(block.code, page)()++ anchor.click.assert_awaited_once_with()+ assert await anchor.is_checked() is True+ assert parent_frame.recaptcha_token._input_value_calls == 2+ agent_function.auto_solve_captchas.assert_not_awaited()+ agent_function.solve_recaptcha_token.assert_not_awaited()+++ "anchor_url",+ (+ "https://evil.example/recaptcha/api2/anchor",+ "https://attacker.storage.googleapis.com/recaptcha/api2/anchor",+ ),+)+async def test_real_sandbox_solve_captcha_does_not_click_untrusted_anchor_host(+ monkeypatch: pytest.MonkeyPatch,+ anchor_url: str,+) -> None:+ agent_function = type(+ "AgentFunctionStub",+ (),+ {+ "auto_solve_captchas": AsyncMock(return_value=True),+ "solve_recaptcha_token": AsyncMock(return_value=False),+ },+ )()+ monkeypatch.setattr(app, "AGENT_FUNCTION", agent_function)+ anchor = FakeLocator(count=1)+ page = FakePage(+ recaptcha=True,+ frames=[FakeFrame(url=anchor_url, anchor=anchor)],+ )+ block = CodeBlock.model_construct(code="await solve_captcha(page)", label="captcha_untrusted_anchor")++ await block.generate_async_user_function(block.code, page)()++ anchor.click.assert_not_awaited()+ agent_function.auto_solve_captchas.assert_awaited_once_with(page)+++async def test_real_sandbox_solve_captcha_does_not_click_after_anchor_frame_navigates(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ agent_function = type(+ "AgentFunctionStub",+ (),+ {+ "auto_solve_captchas": AsyncMock(return_value=True),+ "solve_recaptcha_token": AsyncMock(return_value=False),+ },+ )()+ monkeypatch.setattr(app, "AGENT_FUNCTION", agent_function)+ anchor = FakeLocator(count=1)+ frame = FakeFrame(url="https://www.google.com/recaptcha/api2/anchor", anchor=anchor)++ async def navigate_before_handle_returns() -> FakeLocator:+ frame.url = "https://evil.example/recaptcha/api2/anchor"+ return anchor++ anchor.element_handle = AsyncMock(side_effect=navigate_before_handle_returns)+ page = FakePage(recaptcha=True, frames=[frame])+ block = CodeBlock.model_construct(code="await solve_captcha(page)", label="captcha_navigated_anchor")++ await block.generate_async_user_function(block.code, page)()++ anchor.click.assert_not_awaited()+ agent_function.auto_solve_captchas.assert_awaited_once_with(page)+++ ("destination_url", "navigation_solved"),+ (+ ("https://app.example/account", True),+ ("https://app.example/login#done", False),+ ),+)+async def test_real_sandbox_solve_captcha_returns_after_callback_navigation(+ monkeypatch: pytest.MonkeyPatch,+ destination_url: str,+ navigation_solved: bool,+) -> None:+ agent_function = type(+ "AgentFunctionStub",+ (),+ {+ "auto_solve_captchas": AsyncMock(return_value=not navigation_solved),+ "solve_recaptcha_token": AsyncMock(return_value=False),+ },+ )()+ monkeypatch.setattr(app, "AGENT_FUNCTION", agent_function)+ anchor = FakeLocator(count=1)+ parent_frame = FakePage(token_values=[""])+ frame = FakeFrame(+ url="https://www.google.com/recaptcha/api2/anchor",+ anchor=anchor,+ parent_frame=parent_frame,+ )+ page = FakePage(recaptcha=True, frames=[frame])++ async def navigate_after_click(_milliseconds: int) -> None:+ frame.detached = True+ page.url = destination_url++ page.wait_for_timeout = AsyncMock(side_effect=navigate_after_click)+ block = CodeBlock.model_construct(code="await solve_captcha(page)", label="captcha_callback_navigation")++ await block.generate_async_user_function(block.code, page)()++ anchor.click.assert_awaited_once_with()+ if navigation_solved:+ agent_function.auto_solve_captchas.assert_not_awaited()+ else:+ agent_function.auto_solve_captchas.assert_awaited_once_with(page)+ agent_function.solve_recaptcha_token.assert_not_awaited()+++async def test_real_sandbox_solve_captcha_rejects_prechecked_anchor_without_fresh_token(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ agent_function = type(+ "AgentFunctionStub",+ (),+ {+ "auto_solve_captchas": AsyncMock(return_value=False),+ "solve_recaptcha_token": AsyncMock(return_value=True),+ },+ )()+ monkeypatch.setattr(app, "AGENT_FUNCTION", agent_function)+ anchor = FakeLocator(count=1, checked=True)+ page = FakePage(+ recaptcha=True,+ token_values=["opaque-token"],+ frames=[FakeFrame(url="https://www.google.com/recaptcha/api2/anchor", anchor=anchor)],+ )+ block = CodeBlock.model_construct(code="await solve_captcha(page)", label="captcha_anchor_prechecked")++ await block.generate_async_user_function(block.code, page)()++ anchor.click.assert_not_awaited()+ agent_function.auto_solve_captchas.assert_awaited_once_with(page)+ agent_function.solve_recaptcha_token.assert_awaited_once()+++async def test_real_sandbox_solve_captcha_falls_through_when_anchor_stays_unchecked(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ agent_function = type(+ "AgentFunctionStub",+ (),+ {+ "auto_solve_captchas": AsyncMock(return_value=True),+ "solve_recaptcha_token": AsyncMock(return_value=False),+ },+ )()+ monkeypatch.setattr(app, "AGENT_FUNCTION", agent_function)+ anchor = FakeLocator(count=1)+ anchor.click = AsyncMock()+ page = FakePage(+ recaptcha=True,+ frames=[FakeFrame(url="https://www.google.com/recaptcha/api2/anchor", anchor=anchor)],+ )+ block = CodeBlock.model_construct(code="await solve_captcha(page)", label="captcha_anchor_unchecked")++ await block.generate_async_user_function(block.code, page)()++ anchor.click.assert_awaited_once_with()+ agent_function.auto_solve_captchas.assert_awaited_once_with(page)+ agent_function.solve_recaptcha_token.assert_not_awaited()+++async def test_real_sandbox_solve_captcha_falls_through_when_token_lags_checked_anchor(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ agent_function = type(+ "AgentFunctionStub",+ (),+ {+ "auto_solve_captchas": AsyncMock(return_value=False),+ "solve_recaptcha_token": AsyncMock(return_value=True),+ },+ )()+ monkeypatch.setattr(app, "AGENT_FUNCTION", agent_function)+ anchor = FakeLocator(count=1)+ page = FakePage(+ recaptcha=True,+ token_values=["", ""],+ frames=[FakeFrame(url="https://www.google.com/recaptcha/api2/anchor", anchor=anchor)],+ )+ block = CodeBlock.model_construct(code="await solve_captcha(page)", label="captcha_anchor_late_token")++ await block.generate_async_user_function(block.code, page)()++ assert await anchor.is_checked() is True+ agent_function.auto_solve_captchas.assert_awaited_once_with(page)+ agent_function.solve_recaptcha_token.assert_awaited_once()+++async def test_real_sandbox_solve_captcha_rejects_preexisting_token(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ agent_function = type(+ "AgentFunctionStub",+ (),+ {+ "auto_solve_captchas": AsyncMock(return_value=False),+ "solve_recaptcha_token": AsyncMock(return_value=True),+ },+ )()+ monkeypatch.setattr(app, "AGENT_FUNCTION", agent_function)+ anchor = FakeLocator(count=1)+ page = FakePage(+ recaptcha=True,+ token_values=["stale-token"],+ frames=[FakeFrame(url="https://www.google.com/recaptcha/api2/anchor", anchor=anchor)],+ )+ block = CodeBlock.model_construct(code="await solve_captcha(page)", label="captcha_anchor_stale_token")++ await block.generate_async_user_function(block.code, page)()++ anchor.click.assert_awaited_once_with()+ agent_function.auto_solve_captchas.assert_awaited_once_with(page)+ agent_function.solve_recaptcha_token.assert_awaited_once()+++async def test_real_sandbox_solve_captcha_rejects_inconclusive_token_baseline(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ agent_function = type(+ "AgentFunctionStub",+ (),+ {+ "auto_solve_captchas": AsyncMock(return_value=False),+ "solve_recaptcha_token": AsyncMock(return_value=True),+ },+ )()+ monkeypatch.setattr(app, "AGENT_FUNCTION", agent_function)+ anchor = FakeLocator(count=1)+ page = FakePage(+ recaptcha=True,+ token_values=["opaque-token"],+ frames=[FakeFrame(url="https://www.google.com/recaptcha/api2/anchor", anchor=anchor)],+ )+ page.recaptcha_token.input_value = AsyncMock(+ side_effect=[block_module.PlaywrightError("probe failed"), "opaque-token"]+ )+ block = CodeBlock.model_construct(code="await solve_captcha(page)", label="captcha_anchor_unknown_baseline")++ await block.generate_async_user_function(block.code, page)()++ anchor.click.assert_awaited_once_with()+ agent_function.auto_solve_captchas.assert_awaited_once_with(page)+ agent_function.solve_recaptcha_token.assert_awaited_once()+++async def test_real_sandbox_solve_captcha_bounds_unresponsive_anchor_arm(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ agent_function = type(+ "AgentFunctionStub",+ (),+ {+ "auto_solve_captchas": AsyncMock(return_value=True),+ "solve_recaptcha_token": AsyncMock(return_value=False),+ },+ )()+ monkeypatch.setattr(app, "AGENT_FUNCTION", agent_function)+ monkeypatch.setattr(block_module, "_CODE_BLOCK_RECAPTCHA_ANCHOR_ARM_TIMEOUT_SECONDS", 0.01)++ async def hang() -> None:+ await asyncio.sleep(60)++ anchor = FakeLocator(count=1)+ anchor.click = AsyncMock(side_effect=hang)+ page = FakePage(+ recaptcha=True,+ frames=[FakeFrame(url="https://www.google.com/recaptcha/api2/anchor", anchor=anchor)],+ )+ block = CodeBlock.model_construct(code="await solve_captcha(page)", label="captcha_anchor_timeout")++ await asyncio.wait_for(block.generate_async_user_function(block.code, page)(), timeout=0.25)++ anchor.click.assert_awaited_once_with()+ agent_function.auto_solve_captchas.assert_awaited_once_with(page)+ agent_function.solve_recaptcha_token.assert_not_awaited()++@pytest.mark.asyncioasync def test_real_sandbox_solve_captcha_raises_constant_sanitized_error(monkeypatch: pytest.MonkeyPatch,