🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev
Python aiai-scrapingautomationcrawlercrawlingcrawling-pythondatadata-extractionmcpmcp-serverplaywrightpythonscrapingselectorsstealthweb-scraperweb-scrapingweb-scraping-pythonwebscrapingxpath
fix(solver): fix detection of delayed Cloudflare Turnstile widgets
Karim shoaire60ea5861043
diff --git a/scrapling/engines/_browsers/_stealth.py b/scrapling/engines/_browsers/_stealth.pyindex 225f032..37a4686 100644--- a/scrapling/engines/_browsers/_stealth.py+++ b/scrapling/engines/_browsers/_stealth.py@@ -1,9 +1,9 @@from random import randintfrom re import compile as re_compile-from time import sleep as time_sleep+from time import monotonic, sleep as time_sleepfrom asyncio import sleep as asyncio_sleep-from patchright.sync_api import Locator, Page+from patchright.sync_api import Locator, Page, TimeoutError as PlaywrightTimeoutErrorfrom patchright.async_api import Page as async_Page, Locator as AsyncLocatorfrom patchright.sync_api import sync_playwrightfrom patchright.async_api import async_playwright@@ -118,8 +118,19 @@ class StealthySession(SyncSession, StealthySessionMixin)::param _attempts: The number of solve attempts done so far, used internally to cap the retries.:return:"""- self._wait_for_networkidle(page, timeout=5000)- challenge_type = self._detect_cloudflare(ResponseFactory._get_page_content(page))+ if _attempts:+ self._wait_for_networkidle(page, timeout=5000)+ deadline = monotonic() + 5+ challenge_type = None+ for poll in range(51):+ if not poll % 5 or monotonic() >= deadline:+ challenge_type = self._detect_cloudflare(ResponseFactory._get_page_content(page))+ if challenge_type and (challenge_type != "embedded" or page.frame(url=__CF_PATTERN__) is not None):+ break+ if _attempts or poll == 50 or (remaining := deadline - monotonic()) <= 0:+ challenge_type = None+ break+ page.wait_for_timeout(min(100, remaining * 1000))if not challenge_type:log.info("No Cloudflare challenge found.")return None@@ -128,6 +139,18 @@ class StealthySession(SyncSession, StealthySessionMixin):return Noneelse:log.info(f'The turnstile version discovered is "{challenge_type}"')+ if challenge_type == "embedded":+ iframe = page.frame(url=__CF_PATTERN__)+ if iframe is None:+ return None+ self._wait_for_page_stability(iframe, True, False)+ try:+ iframe.locator('input[type="checkbox"]').click(timeout=10000, delay=randint(100, 200))+ except PlaywrightTimeoutError:+ return None+ self._wait_for_networkidle(page, timeout=5000)+ self._wait_for_page_stability(page, True, False)+ return Noneif challenge_type == "non-interactive":for _ in range(10):if self._detect_cloudflare(ResponseFactory._get_page_content(page)) != "non-interactive":@@ -412,8 +435,19 @@ class AsyncStealthySession(AsyncSession, StealthySessionMixin)::param _attempts: The number of solve attempts done so far, used internally to cap the retries.:return:"""- await self._wait_for_networkidle(page, timeout=5000)- challenge_type = self._detect_cloudflare(await ResponseFactory._get_async_page_content(page))+ if _attempts:+ await self._wait_for_networkidle(page, timeout=5000)+ deadline = monotonic() + 5+ challenge_type = None+ for poll in range(51):+ if not poll % 5 or monotonic() >= deadline:+ challenge_type = self._detect_cloudflare(await ResponseFactory._get_async_page_content(page))+ if challenge_type and (challenge_type != "embedded" or page.frame(url=__CF_PATTERN__) is not None):+ break+ if _attempts or poll == 50 or (remaining := deadline - monotonic()) <= 0:+ challenge_type = None+ break+ await page.wait_for_timeout(min(100, remaining * 1000))if not challenge_type:log.info("No Cloudflare challenge found.")return None@@ -422,6 +456,18 @@ class AsyncStealthySession(AsyncSession, StealthySessionMixin):return Noneelse:log.info(f'The turnstile version discovered is "{challenge_type}"')+ if challenge_type == "embedded":+ iframe = page.frame(url=__CF_PATTERN__)+ if iframe is None:+ return None+ await self._wait_for_page_stability(iframe, True, False)+ try:+ await iframe.locator('input[type="checkbox"]').click(timeout=10000, delay=randint(100, 200))+ except PlaywrightTimeoutError:+ return None+ await self._wait_for_networkidle(page, timeout=5000)+ await self._wait_for_page_stability(page, True, False)+ return Noneif challenge_type == "non-interactive":for _ in range(10):if (diff --git a/tests/fetchers/test_cloudflare_polling.py b/tests/fetchers/test_cloudflare_polling.pynew file mode 100644index 0000000..a34dbc2--- /dev/null+++ b/tests/fetchers/test_cloudflare_polling.py@@ -0,0 +1,135 @@+import asyncio+from unittest.mock import AsyncMock, MagicMock++import pytest+from patchright.sync_api import Error as PlaywrightError, TimeoutError as PlaywrightTimeoutError++from scrapling.engines._browsers import _stealth+++EMBEDDED = '<script src="https://challenges.cloudflare.com/turnstile/v0/api.js"></script>'+++def setup(monkeypatch, asynchronous, script_at=0, frame_at=0, content_cost=0):+ mock = AsyncMock if asynchronous else MagicMock+ session_type = _stealth.AsyncStealthySession if asynchronous else _stealth.StealthySession+ monkeypatch.setattr(session_type, "_wait_for_networkidle", mock())+ monkeypatch.setattr(session_type, "_wait_for_page_stability", mock())+ session = object.__new__(session_type)+ clock = {"elapsed": 0.0}+ monkeypatch.setattr(_stealth, "monotonic", lambda: clock["elapsed"] / 1000)+ page, frame = MagicMock(), MagicMock()++ def content():+ clock["elapsed"] += content_cost+ return EMBEDDED if script_at is not None and clock["elapsed"] >= script_at else "<p>Page</p>"++ def sleep(milliseconds):+ clock["elapsed"] = round(clock["elapsed"] + milliseconds, 6)++ page.content = mock(side_effect=content)+ page.frame.side_effect = lambda **kwargs: frame if frame_at is not None and clock["elapsed"] >= frame_at else None+ page.wait_for_timeout = mock(side_effect=sleep)+ frame.locator.return_value.click = mock()+ return session, page, frame, clock+++def solve(session, page, asynchronous, attempts=0):+ result = session._cloudflare_solver(page, attempts)+ if asynchronous:+ asyncio.run(result)++[email protected]("asynchronous", [False, True])+ "script_at,frame_at,elapsed,clicks",+ [+ (0, 0, 0, 1),+ (3000, 3000, 3000, 1),+ (0, 750, 800, 1),+ (750, 750, 1000, 1),+ (5000, 5000, 5000, 1),+ (0, 5000, 5000, 1),+ (6000, 6000, 5000, 0),+ (0, 6000, 5000, 0),+ (None, None, 5000, 0),+ (0, None, 5000, 0),+ ],+)+def test_initial_readiness(monkeypatch, asynchronous, script_at, frame_at, elapsed, clicks):+ session, page, frame, clock = setup(monkeypatch, asynchronous, script_at, frame_at)+ solve(session, page, asynchronous)+ assert clock["elapsed"] == elapsed+ assert frame.locator.return_value.click.call_count == clicks+ assert page.content.call_count == elapsed // 500 + 1+ if clicks:+ frame.locator.assert_called_once_with('input[type="checkbox"]')+ assert frame.locator.return_value.click.call_args.kwargs["timeout"] == 10000+ session._wait_for_networkidle.assert_called_once_with(page, timeout=5000)+ else:+ session._wait_for_networkidle.assert_not_called()+ session._wait_for_page_stability.assert_not_called()+ page.mouse.click.assert_not_called()++[email protected]("asynchronous", [False, True])[email protected]("content_cost", [250, 6000])+def test_html_reading_uses_the_same_wait_budget(monkeypatch, asynchronous, content_cost):+ session, page, frame, clock = setup(monkeypatch, asynchronous, None, None, content_cost)+ solve(session, page, asynchronous)+ assert clock["elapsed"] <= max(5000 + content_cost, content_cost)+ assert sum(call.args[0] for call in page.wait_for_timeout.call_args_list) < 5000+ session._wait_for_networkidle.assert_not_called()+ frame.locator.return_value.click.assert_not_called()++[email protected]("asynchronous", [False, True])+def test_final_snapshot_is_refreshed_after_slow_html_reads(monkeypatch, asynchronous):+ session, page, frame, clock = setup(monkeypatch, asynchronous, 4950, 4950, content_cost=120)+ solve(session, page, asynchronous)+ assert 5000 <= clock["elapsed"] <= 5120+ frame.locator.return_value.click.assert_called_once()++[email protected]("asynchronous", [False, True])+def test_recursive_check_has_no_new_polling_window(monkeypatch, asynchronous):+ session, page, frame, clock = setup(monkeypatch, asynchronous, 0, None)+ solve(session, page, asynchronous, attempts=1)+ assert clock["elapsed"] == 0+ page.wait_for_timeout.assert_not_called()+ session._wait_for_networkidle.assert_called_once_with(page, timeout=5000)+ frame.locator.return_value.click.assert_not_called()++[email protected]("asynchronous", [False, True])+def test_attached_frame_keeps_separate_load_and_checkbox_waits(monkeypatch, asynchronous):+ session, page, frame, clock = setup(monkeypatch, asynchronous)++ def wait(target, *args):+ if target is frame:+ clock["elapsed"] += 11000++ session._wait_for_page_stability.side_effect = wait+ frame.locator.return_value.click.side_effect = lambda **kwargs: clock.update(elapsed=clock["elapsed"] + 6000)+ solve(session, page, asynchronous)+ assert clock["elapsed"] == 17000+ frame.locator.return_value.click.assert_called_once()+ page.wait_for_timeout.assert_not_called()++[email protected]("asynchronous", [False, True])[email protected]("missing", [False, True])+def test_only_checkbox_timeout_is_optional(monkeypatch, asynchronous, missing):+ session, page, frame, clock = setup(monkeypatch, asynchronous)+ frame.locator.return_value.click.side_effect = (+ PlaywrightTimeoutError("No checkbox") if missing else PlaywrightError("Page closed")+ )+ logger = MagicMock()+ monkeypatch.setattr(_stealth, "log", logger)+ if missing:+ solve(session, page, asynchronous)+ else:+ with pytest.raises(PlaywrightError, match="Page closed"):+ solve(session, page, asynchronous)+ assert all(call.args != ("Cloudflare captcha is solved",) for call in logger.info.call_args_list)+ session._wait_for_networkidle.assert_not_called()diff --git a/tests/fetchers/test_cloudflare_readiness.py b/tests/fetchers/test_cloudflare_readiness.pynew file mode 100644index 0000000..f8fcac7--- /dev/null+++ b/tests/fetchers/test_cloudflare_readiness.py@@ -0,0 +1,158 @@+from os import getenv+from time import monotonic++import pytest+from patchright.async_api import Page as AsyncPage, Route as AsyncRoute+from patchright.sync_api import Page, Route++from scrapling.core._types import Any+from scrapling.fetchers import AsyncStealthySession, StealthySession+++URL = "https://cloudflare-readiness.test/"+FRAME_URL = "https://challenges.cloudflare.com/cdn-cgi/challenge-platform/test/widget"+SCRIPT_URL = "https://challenges.cloudflare.com/turnstile/v0/api.js"+PAGE = """<!DOCTYPE html><html><head><link rel="icon" href="data:,"></head><body>+<script>addEventListener('message', event => {+ if (event.data === 'solved') {+ document.body.dataset.clicks = Number(document.body.dataset.clicks || 0) + 1;+ document.body.dataset.clickedAt = performance.now();+ }+});</script></body></html>"""+CHECKBOX = """<input type="checkbox" style="width:24px;height:24px;margin:16px"+onclick="this.dataset.clicks = Number(this.dataset.clicks || 0) + 1; parent.postMessage('solved', '*')">"""+WIDGET = f'<!DOCTYPE html><html><body style="margin:0">{CHECKBOX}</body></html>'+DELAYED_WIDGET = f"""<!DOCTYPE html><html><body style="margin:0"><script>+setTimeout(() => document.body.insertAdjacentHTML('beforeend', `{CHECKBOX}`), 700);+</script></body></html>"""+ADD_SCRIPT = f"""const script = document.createElement('script');+script.onload = () => document.body.dataset.scriptLoaded = 'true';+script.src = '{SCRIPT_URL}'; document.head.appendChild(script);"""+ADD_FRAME = f"""const frame = document.createElement('iframe');+frame.style = 'border:0;width:300px;height:150px'; frame.src = '{FRAME_URL}'; document.body.appendChild(frame);"""+SCRIPT = f'<script src="{SCRIPT_URL}"></script>'+FRAME = f'<iframe style="border:0;width:300px;height:150px" src="{FRAME_URL}"></iframe>'+BUSY = """<script>setInterval(() => fetch('/noise?' + Math.random()).then(response => response.text()).then(() => {+ document.body.dataset.background = Number(document.body.dataset.background || 0) + 1;+}), 100);</script>"""+++def options() -> dict[str, Any]:+ return {+ "executable_path": getenv("SCRAPLING_EXECUTABLE_PATH"),+ "google_search": False,+ "solve_cloudflare": True,+ "retries": 1,+ }+++def route_payload(url: str) -> tuple[str, str]:+ if url == SCRIPT_URL:+ return "application/javascript", ""+ if url.startswith(FRAME_URL):+ return "text/html", DELAYED_WIDGET if url.endswith("?delayed") else WIDGET+ if url.startswith(URL + "noise?"):+ return "text/plain", "ok"+ content = {+ "delayed_script": f"<script>setTimeout(() => {{{ADD_SCRIPT}}}, 700); setTimeout(() => {{{ADD_FRAME}}}, 1300);</script>",+ "delayed_frame": SCRIPT + f"<script>setTimeout(() => {{{ADD_FRAME}}}, 900);</script>",+ "initial_checkbox": SCRIPT + FRAME.replace(FRAME_URL, FRAME_URL + "?delayed"),+ "busy": SCRIPT + FRAME + BUSY,+ }.get(url.removeprefix(URL), "")+ return "text/html", PAGE.replace("</body>", content + "</body>")+++def setup_page(page: Page) -> None:+ def fulfill(route: Route) -> None:+ content_type, body = route_payload(route.request.url)+ route.fulfill(content_type=content_type, body=body)++ page.route("**/*", fulfill)+++async def setup_async_page(page: AsyncPage) -> None:+ async def fulfill(route: AsyncRoute) -> None:+ content_type, body = route_payload(route.request.url)+ await route.fulfill(content_type=content_type, body=body)++ await page.route("**/*", fulfill)++[email protected]("case", ["delayed_script", "delayed_frame", "initial_checkbox", "busy"])+def test_sync_widget_readiness(case: str) -> None:+ with StealthySession(**options()) as session:+ started = monotonic()+ response = session.fetch(URL + case, page_setup=setup_page)+ elapsed = monotonic() - started+ assert response.css('body[data-clicks="1"]')+ assert float(response.css("body")[0].attrib["data-clicked-at"]) < 3500+ page = session.page_pool.pages[0].page+ checkbox = page.frame_locator("iframe").locator('input[type="checkbox"]')+ assert checkbox.is_checked()+ assert checkbox.get_attribute("data-clicks") == "1"+ if case == "busy":+ assert 4.8 <= elapsed < 8, elapsed+ assert int(response.css("body")[0].attrib.get("data-background", "0")) >= 2+ elif case == "delayed_script":+ assert response.css('body[data-script-loaded="true"]')+ assert elapsed < 8, elapsed+ else:+ assert elapsed < 4.5, elapsed++ assert page.is_closed()++[email protected]("case", ["delayed_script", "delayed_frame", "initial_checkbox", "busy"])+async def test_async_widget_readiness(case: str) -> None:+ async with AsyncStealthySession(**options()) as session:+ started = monotonic()+ response = await session.fetch(URL + case, page_setup=setup_async_page)+ elapsed = monotonic() - started+ assert response.css('body[data-clicks="1"]')+ assert float(response.css("body")[0].attrib["data-clicked-at"]) < 3500+ page = session.page_pool.pages[0].page+ checkbox = page.frame_locator("iframe").locator('input[type="checkbox"]')+ assert await checkbox.is_checked()+ assert await checkbox.get_attribute("data-clicks") == "1"+ if case == "busy":+ assert 4.8 <= elapsed < 8, elapsed+ assert int(response.css("body")[0].attrib.get("data-background", "0")) >= 2+ elif case == "delayed_script":+ assert response.css('body[data-script-loaded="true"]')+ assert elapsed < 8, elapsed+ else:+ assert elapsed < 4.5, elapsed++ assert page.is_closed()++[email protected]("enabled", [False, True])+def test_sync_clean_page_grace_period(enabled: bool) -> None:+ with StealthySession(**options()) as session:+ started = monotonic()+ response = session.fetch(URL + "clean", solve_cloudflare=enabled, page_setup=setup_page)+ elapsed = monotonic() - started+ assert response.status == 200 and not response.css("iframe")+ assert (4.8 <= elapsed < 7.5) if enabled else elapsed < 3, elapsed+ page = session.page_pool.pages[0].page++ assert page.is_closed()++[email protected]("enabled", [False, True])+async def test_async_clean_page_grace_period(enabled: bool) -> None:+ async with AsyncStealthySession(**options()) as session:+ started = monotonic()+ response = await session.fetch(URL + "clean", solve_cloudflare=enabled, page_setup=setup_async_page)+ elapsed = monotonic() - started+ assert response.status == 200 and not response.css("iframe")+ assert (4.8 <= elapsed < 7.5) if enabled else elapsed < 3, elapsed+ page = session.page_pool.pages[0].page++ assert page.is_closed()