Skip to content

🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev

fix(solver): fix detection of delayed Cloudflare Turnstile widgets

Karim shoaire60ea5861043
diff --git a/scrapling/engines/_browsers/_stealth.py b/scrapling/engines/_browsers/_stealth.py
index 225f032..37a4686 100644
--- a/scrapling/engines/_browsers/_stealth.py
+++ b/scrapling/engines/_browsers/_stealth.py
@@ -1,9 +1,9 @@
from random import randint
from re import compile as re_compile
-from time import sleep as time_sleep
+from time import monotonic, sleep as time_sleep
from asyncio import sleep as asyncio_sleep
-from patchright.sync_api import Locator, Page
+from patchright.sync_api import Locator, Page, TimeoutError as PlaywrightTimeoutError
from patchright.async_api import Page as async_Page, Locator as AsyncLocator
from patchright.sync_api import sync_playwright
from patchright.async_api import async_playwright
@@ -118,8 +118,19 @@ class StealthySession(SyncSession, StealthySessionMixin):
:param _attempts: The number of solve attempts done so far, used internally to cap the retries.
:return:
"""
- self._wait_for_networkidle(page, timeout=5000)
- challenge_type = self._detect_cloudflare(ResponseFactory._get_page_content(page))
+ if _attempts:
+ self._wait_for_networkidle(page, timeout=5000)
+ deadline = monotonic() + 5
+ challenge_type = None
+ for poll in range(51):
+ if not poll % 5 or monotonic() >= deadline:
+ challenge_type = self._detect_cloudflare(ResponseFactory._get_page_content(page))
+ if challenge_type and (challenge_type != "embedded" or page.frame(url=__CF_PATTERN__) is not None):
+ break
+ if _attempts or poll == 50 or (remaining := deadline - monotonic()) <= 0:
+ challenge_type = None
+ break
+ page.wait_for_timeout(min(100, remaining * 1000))
if not challenge_type:
log.info("No Cloudflare challenge found.")
return None
@@ -128,6 +139,18 @@ class StealthySession(SyncSession, StealthySessionMixin):
return None
else:
log.info(f'The turnstile version discovered is "{challenge_type}"')
+ if challenge_type == "embedded":
+ iframe = page.frame(url=__CF_PATTERN__)
+ if iframe is None:
+ return None
+ self._wait_for_page_stability(iframe, True, False)
+ try:
+ iframe.locator('input[type="checkbox"]').click(timeout=10000, delay=randint(100, 200))
+ except PlaywrightTimeoutError:
+ return None
+ self._wait_for_networkidle(page, timeout=5000)
+ self._wait_for_page_stability(page, True, False)
+ return None
if challenge_type == "non-interactive":
for _ in range(10):
if self._detect_cloudflare(ResponseFactory._get_page_content(page)) != "non-interactive":
@@ -412,8 +435,19 @@ class AsyncStealthySession(AsyncSession, StealthySessionMixin):
:param _attempts: The number of solve attempts done so far, used internally to cap the retries.
:return:
"""
- await self._wait_for_networkidle(page, timeout=5000)
- challenge_type = self._detect_cloudflare(await ResponseFactory._get_async_page_content(page))
+ if _attempts:
+ await self._wait_for_networkidle(page, timeout=5000)
+ deadline = monotonic() + 5
+ challenge_type = None
+ for poll in range(51):
+ if not poll % 5 or monotonic() >= deadline:
+ challenge_type = self._detect_cloudflare(await ResponseFactory._get_async_page_content(page))
+ if challenge_type and (challenge_type != "embedded" or page.frame(url=__CF_PATTERN__) is not None):
+ break
+ if _attempts or poll == 50 or (remaining := deadline - monotonic()) <= 0:
+ challenge_type = None
+ break
+ await page.wait_for_timeout(min(100, remaining * 1000))
if not challenge_type:
log.info("No Cloudflare challenge found.")
return None
@@ -422,6 +456,18 @@ class AsyncStealthySession(AsyncSession, StealthySessionMixin):
return None
else:
log.info(f'The turnstile version discovered is "{challenge_type}"')
+ if challenge_type == "embedded":
+ iframe = page.frame(url=__CF_PATTERN__)
+ if iframe is None:
+ return None
+ await self._wait_for_page_stability(iframe, True, False)
+ try:
+ await iframe.locator('input[type="checkbox"]').click(timeout=10000, delay=randint(100, 200))
+ except PlaywrightTimeoutError:
+ return None
+ await self._wait_for_networkidle(page, timeout=5000)
+ await self._wait_for_page_stability(page, True, False)
+ return None
if challenge_type == "non-interactive":
for _ in range(10):
if (
diff --git a/tests/fetchers/test_cloudflare_polling.py b/tests/fetchers/test_cloudflare_polling.py
new file mode 100644
index 0000000..a34dbc2
--- /dev/null
+++ b/tests/fetchers/test_cloudflare_polling.py
@@ -0,0 +1,135 @@
+import asyncio
+from unittest.mock import AsyncMock, MagicMock
+
+import pytest
+from patchright.sync_api import Error as PlaywrightError, TimeoutError as PlaywrightTimeoutError
+
+from scrapling.engines._browsers import _stealth
+
+
+EMBEDDED = '<script src="https://challenges.cloudflare.com/turnstile/v0/api.js"></script>'
+
+
+def setup(monkeypatch, asynchronous, script_at=0, frame_at=0, content_cost=0):
+ mock = AsyncMock if asynchronous else MagicMock
+ session_type = _stealth.AsyncStealthySession if asynchronous else _stealth.StealthySession
+ monkeypatch.setattr(session_type, "_wait_for_networkidle", mock())
+ monkeypatch.setattr(session_type, "_wait_for_page_stability", mock())
+ session = object.__new__(session_type)
+ clock = {"elapsed": 0.0}
+ monkeypatch.setattr(_stealth, "monotonic", lambda: clock["elapsed"] / 1000)
+ page, frame = MagicMock(), MagicMock()
+
+ def content():
+ clock["elapsed"] += content_cost
+ return EMBEDDED if script_at is not None and clock["elapsed"] >= script_at else "<p>Page</p>"
+
+ def sleep(milliseconds):
+ clock["elapsed"] = round(clock["elapsed"] + milliseconds, 6)
+
+ page.content = mock(side_effect=content)
+ page.frame.side_effect = lambda **kwargs: frame if frame_at is not None and clock["elapsed"] >= frame_at else None
+ page.wait_for_timeout = mock(side_effect=sleep)
+ frame.locator.return_value.click = mock()
+ return session, page, frame, clock
+
+
+def solve(session, page, asynchronous, attempts=0):
+ result = session._cloudflare_solver(page, attempts)
+ if asynchronous:
+ asyncio.run(result)
+
+
[email protected]("asynchronous", [False, True])
+ "script_at,frame_at,elapsed,clicks",
+ [
+ (0, 0, 0, 1),
+ (3000, 3000, 3000, 1),
+ (0, 750, 800, 1),
+ (750, 750, 1000, 1),
+ (5000, 5000, 5000, 1),
+ (0, 5000, 5000, 1),
+ (6000, 6000, 5000, 0),
+ (0, 6000, 5000, 0),
+ (None, None, 5000, 0),
+ (0, None, 5000, 0),
+ ],
+)
+def test_initial_readiness(monkeypatch, asynchronous, script_at, frame_at, elapsed, clicks):
+ session, page, frame, clock = setup(monkeypatch, asynchronous, script_at, frame_at)
+ solve(session, page, asynchronous)
+ assert clock["elapsed"] == elapsed
+ assert frame.locator.return_value.click.call_count == clicks
+ assert page.content.call_count == elapsed // 500 + 1
+ if clicks:
+ frame.locator.assert_called_once_with('input[type="checkbox"]')
+ assert frame.locator.return_value.click.call_args.kwargs["timeout"] == 10000
+ session._wait_for_networkidle.assert_called_once_with(page, timeout=5000)
+ else:
+ session._wait_for_networkidle.assert_not_called()
+ session._wait_for_page_stability.assert_not_called()
+ page.mouse.click.assert_not_called()
+
+
[email protected]("asynchronous", [False, True])
[email protected]("content_cost", [250, 6000])
+def test_html_reading_uses_the_same_wait_budget(monkeypatch, asynchronous, content_cost):
+ session, page, frame, clock = setup(monkeypatch, asynchronous, None, None, content_cost)
+ solve(session, page, asynchronous)
+ assert clock["elapsed"] <= max(5000 + content_cost, content_cost)
+ assert sum(call.args[0] for call in page.wait_for_timeout.call_args_list) < 5000
+ session._wait_for_networkidle.assert_not_called()
+ frame.locator.return_value.click.assert_not_called()
+
+
[email protected]("asynchronous", [False, True])
+def test_final_snapshot_is_refreshed_after_slow_html_reads(monkeypatch, asynchronous):
+ session, page, frame, clock = setup(monkeypatch, asynchronous, 4950, 4950, content_cost=120)
+ solve(session, page, asynchronous)
+ assert 5000 <= clock["elapsed"] <= 5120
+ frame.locator.return_value.click.assert_called_once()
+
+
[email protected]("asynchronous", [False, True])
+def test_recursive_check_has_no_new_polling_window(monkeypatch, asynchronous):
+ session, page, frame, clock = setup(monkeypatch, asynchronous, 0, None)
+ solve(session, page, asynchronous, attempts=1)
+ assert clock["elapsed"] == 0
+ page.wait_for_timeout.assert_not_called()
+ session._wait_for_networkidle.assert_called_once_with(page, timeout=5000)
+ frame.locator.return_value.click.assert_not_called()
+
+
[email protected]("asynchronous", [False, True])
+def test_attached_frame_keeps_separate_load_and_checkbox_waits(monkeypatch, asynchronous):
+ session, page, frame, clock = setup(monkeypatch, asynchronous)
+
+ def wait(target, *args):
+ if target is frame:
+ clock["elapsed"] += 11000
+
+ session._wait_for_page_stability.side_effect = wait
+ frame.locator.return_value.click.side_effect = lambda **kwargs: clock.update(elapsed=clock["elapsed"] + 6000)
+ solve(session, page, asynchronous)
+ assert clock["elapsed"] == 17000
+ frame.locator.return_value.click.assert_called_once()
+ page.wait_for_timeout.assert_not_called()
+
+
[email protected]("asynchronous", [False, True])
[email protected]("missing", [False, True])
+def test_only_checkbox_timeout_is_optional(monkeypatch, asynchronous, missing):
+ session, page, frame, clock = setup(monkeypatch, asynchronous)
+ frame.locator.return_value.click.side_effect = (
+ PlaywrightTimeoutError("No checkbox") if missing else PlaywrightError("Page closed")
+ )
+ logger = MagicMock()
+ monkeypatch.setattr(_stealth, "log", logger)
+ if missing:
+ solve(session, page, asynchronous)
+ else:
+ with pytest.raises(PlaywrightError, match="Page closed"):
+ solve(session, page, asynchronous)
+ assert all(call.args != ("Cloudflare captcha is solved",) for call in logger.info.call_args_list)
+ session._wait_for_networkidle.assert_not_called()
diff --git a/tests/fetchers/test_cloudflare_readiness.py b/tests/fetchers/test_cloudflare_readiness.py
new file mode 100644
index 0000000..f8fcac7
--- /dev/null
+++ b/tests/fetchers/test_cloudflare_readiness.py
@@ -0,0 +1,158 @@
+from os import getenv
+from time import monotonic
+
+import pytest
+from patchright.async_api import Page as AsyncPage, Route as AsyncRoute
+from patchright.sync_api import Page, Route
+
+from scrapling.core._types import Any
+from scrapling.fetchers import AsyncStealthySession, StealthySession
+
+
+URL = "https://cloudflare-readiness.test/"
+FRAME_URL = "https://challenges.cloudflare.com/cdn-cgi/challenge-platform/test/widget"
+SCRIPT_URL = "https://challenges.cloudflare.com/turnstile/v0/api.js"
+PAGE = """<!DOCTYPE html><html><head><link rel="icon" href="data:,"></head><body>
+<script>addEventListener('message', event => {
+ if (event.data === 'solved') {
+ document.body.dataset.clicks = Number(document.body.dataset.clicks || 0) + 1;
+ document.body.dataset.clickedAt = performance.now();
+ }
+});</script></body></html>"""
+CHECKBOX = """<input type="checkbox" style="width:24px;height:24px;margin:16px"
+onclick="this.dataset.clicks = Number(this.dataset.clicks || 0) + 1; parent.postMessage('solved', '*')">"""
+WIDGET = f'<!DOCTYPE html><html><body style="margin:0">{CHECKBOX}</body></html>'
+DELAYED_WIDGET = f"""<!DOCTYPE html><html><body style="margin:0"><script>
+setTimeout(() => document.body.insertAdjacentHTML('beforeend', `{CHECKBOX}`), 700);
+</script></body></html>"""
+ADD_SCRIPT = f"""const script = document.createElement('script');
+script.onload = () => document.body.dataset.scriptLoaded = 'true';
+script.src = '{SCRIPT_URL}'; document.head.appendChild(script);"""
+ADD_FRAME = f"""const frame = document.createElement('iframe');
+frame.style = 'border:0;width:300px;height:150px'; frame.src = '{FRAME_URL}'; document.body.appendChild(frame);"""
+SCRIPT = f'<script src="{SCRIPT_URL}"></script>'
+FRAME = f'<iframe style="border:0;width:300px;height:150px" src="{FRAME_URL}"></iframe>'
+BUSY = """<script>setInterval(() => fetch('/noise?' + Math.random()).then(response => response.text()).then(() => {
+ document.body.dataset.background = Number(document.body.dataset.background || 0) + 1;
+}), 100);</script>"""
+
+
+def options() -> dict[str, Any]:
+ return {
+ "executable_path": getenv("SCRAPLING_EXECUTABLE_PATH"),
+ "google_search": False,
+ "solve_cloudflare": True,
+ "retries": 1,
+ }
+
+
+def route_payload(url: str) -> tuple[str, str]:
+ if url == SCRIPT_URL:
+ return "application/javascript", ""
+ if url.startswith(FRAME_URL):
+ return "text/html", DELAYED_WIDGET if url.endswith("?delayed") else WIDGET
+ if url.startswith(URL + "noise?"):
+ return "text/plain", "ok"
+ content = {
+ "delayed_script": f"<script>setTimeout(() => {{{ADD_SCRIPT}}}, 700); setTimeout(() => {{{ADD_FRAME}}}, 1300);</script>",
+ "delayed_frame": SCRIPT + f"<script>setTimeout(() => {{{ADD_FRAME}}}, 900);</script>",
+ "initial_checkbox": SCRIPT + FRAME.replace(FRAME_URL, FRAME_URL + "?delayed"),
+ "busy": SCRIPT + FRAME + BUSY,
+ }.get(url.removeprefix(URL), "")
+ return "text/html", PAGE.replace("</body>", content + "</body>")
+
+
+def setup_page(page: Page) -> None:
+ def fulfill(route: Route) -> None:
+ content_type, body = route_payload(route.request.url)
+ route.fulfill(content_type=content_type, body=body)
+
+ page.route("**/*", fulfill)
+
+
+async def setup_async_page(page: AsyncPage) -> None:
+ async def fulfill(route: AsyncRoute) -> None:
+ content_type, body = route_payload(route.request.url)
+ await route.fulfill(content_type=content_type, body=body)
+
+ await page.route("**/*", fulfill)
+
+
[email protected]("case", ["delayed_script", "delayed_frame", "initial_checkbox", "busy"])
+def test_sync_widget_readiness(case: str) -> None:
+ with StealthySession(**options()) as session:
+ started = monotonic()
+ response = session.fetch(URL + case, page_setup=setup_page)
+ elapsed = monotonic() - started
+ assert response.css('body[data-clicks="1"]')
+ assert float(response.css("body")[0].attrib["data-clicked-at"]) < 3500
+ page = session.page_pool.pages[0].page
+ checkbox = page.frame_locator("iframe").locator('input[type="checkbox"]')
+ assert checkbox.is_checked()
+ assert checkbox.get_attribute("data-clicks") == "1"
+ if case == "busy":
+ assert 4.8 <= elapsed < 8, elapsed
+ assert int(response.css("body")[0].attrib.get("data-background", "0")) >= 2
+ elif case == "delayed_script":
+ assert response.css('body[data-script-loaded="true"]')
+ assert elapsed < 8, elapsed
+ else:
+ assert elapsed < 4.5, elapsed
+
+ assert page.is_closed()
+
+
[email protected]("case", ["delayed_script", "delayed_frame", "initial_checkbox", "busy"])
+async def test_async_widget_readiness(case: str) -> None:
+ async with AsyncStealthySession(**options()) as session:
+ started = monotonic()
+ response = await session.fetch(URL + case, page_setup=setup_async_page)
+ elapsed = monotonic() - started
+ assert response.css('body[data-clicks="1"]')
+ assert float(response.css("body")[0].attrib["data-clicked-at"]) < 3500
+ page = session.page_pool.pages[0].page
+ checkbox = page.frame_locator("iframe").locator('input[type="checkbox"]')
+ assert await checkbox.is_checked()
+ assert await checkbox.get_attribute("data-clicks") == "1"
+ if case == "busy":
+ assert 4.8 <= elapsed < 8, elapsed
+ assert int(response.css("body")[0].attrib.get("data-background", "0")) >= 2
+ elif case == "delayed_script":
+ assert response.css('body[data-script-loaded="true"]')
+ assert elapsed < 8, elapsed
+ else:
+ assert elapsed < 4.5, elapsed
+
+ assert page.is_closed()
+
+
[email protected]("enabled", [False, True])
+def test_sync_clean_page_grace_period(enabled: bool) -> None:
+ with StealthySession(**options()) as session:
+ started = monotonic()
+ response = session.fetch(URL + "clean", solve_cloudflare=enabled, page_setup=setup_page)
+ elapsed = monotonic() - started
+ assert response.status == 200 and not response.css("iframe")
+ assert (4.8 <= elapsed < 7.5) if enabled else elapsed < 3, elapsed
+ page = session.page_pool.pages[0].page
+
+ assert page.is_closed()
+
+
[email protected]("enabled", [False, True])
+async def test_async_clean_page_grace_period(enabled: bool) -> None:
+ async with AsyncStealthySession(**options()) as session:
+ started = monotonic()
+ response = await session.fetch(URL + "clean", solve_cloudflare=enabled, page_setup=setup_async_page)
+ elapsed = monotonic() - started
+ assert response.status == 200 and not response.css("iframe")
+ assert (4.8 <= elapsed < 7.5) if enabled else elapsed < 3, elapsed
+ page = session.page_pool.pages[0].page
+
+ assert page.is_closed()