🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev
Python aiai-scrapingautomationcrawlercrawlingcrawling-pythondatadata-extractionmcpmcp-serverplaywrightpythonscrapingselectorsstealthweb-scraperweb-scrapingweb-scraping-pythonwebscrapingxpath
fix(stealth): solve Cloudflare challenges regardless of locale + retry cap + fix crashing mid-solve
Karim shoairc0326263b525
diff --git a/scrapling/engines/_browsers/_base.py b/scrapling/engines/_browsers/_base.pyindex c9d390c..12a44c5 100644--- a/scrapling/engines/_browsers/_base.py+++ b/scrapling/engines/_browsers/_base.py@@ -437,11 +437,13 @@ class BaseSessionMixin:self._context_options.update({"proxy": config.proxy,- "locale": config.locale,"timezone_id": config.timezone_id,"extra_http_headers": config.extra_headers,})+ if config.locale and config.cdp_url:+ # Launch flags can't be set on remote browsers, so the detectable context option is the best effort left+ self._context_options["locale"] = config.locale# The default useragent in the headful is always correct now in the current versions of Playwrightif config.useragent:self._context_options["user_agent"] = config.useragent@@ -462,6 +464,17 @@ class BaseSessionMixin:else:flags = list(flags) + [doh_flag]+ if config.locale:+ # The context `locale` option patches the main thread only, so Web Workers keep the browser's real+ # language and WAFs like Cloudflare flag the mismatch. Launch flags set it browser-wide instead,+ # so workers, `Intl`, and the `Accept-Language` header all follow natively.+ base_lang = config.locale.split("-")[0].lower()+ accept_lang = f"{config.locale},{base_lang}" if base_lang != config.locale.lower() else config.locale+ flags = (flags if isinstance(flags, list) else list(flags)) + [+ f"--lang={config.locale}",+ f"--accept-lang={accept_lang}",+ ]+self._browser_options.update({"args": flags,@@ -575,3 +588,17 @@ class StealthySessionMixin(BaseSessionMixin):return "embedded"return None++ @classmethod+ def _challenge_cleared(cls, page_content: str, challenge_type: str) -> bool:+ """+ Check whether the Cloudflare challenge is no longer present in the page content.++ Args:+ page_content (str): The content of the page to analyze.+ challenge_type (str): The challenge type returned by `_detect_cloudflare`.++ Returns:+ bool: True if the challenge is gone from the page, False otherwise.+ """+ return challenge_type == "embedded" or cls._detect_cloudflare(page_content) is Nonediff --git a/scrapling/engines/_browsers/_stealth.py b/scrapling/engines/_browsers/_stealth.pyindex efa195e..0eb98e3 100644--- a/scrapling/engines/_browsers/_stealth.py+++ b/scrapling/engines/_browsers/_stealth.py@@ -17,6 +17,7 @@ from scrapling.engines._browsers._base import SyncSession, AsyncSession, Stealthfrom scrapling.engines._browsers._validators import validate_fetch as _validate, StealthConfig__CF_PATTERN__ = re_compile(r"^https?://challenges\.cloudflare\.com/cdn-cgi/challenge-platform/.*")+__CF_MAX_SOLVE_ATTEMPTS__ = 3class StealthySession(SyncSession, StealthySessionMixin):@@ -104,10 +105,11 @@ class StealthySession(SyncSession, StealthySessionMixin):else:raise RuntimeError("Session has been already started")- def _cloudflare_solver(self, page: Page) -> None: # pragma: no cover+ def _cloudflare_solver(self, page: Page, _attempts: int = 0) -> None: # pragma: no cover"""Solve the cloudflare challenge displayed on the playwright page passed:param page: The targeted page+ :param _attempts: The number of solve attempts done so far, used internally to cap the retries.:return:"""self._wait_for_networkidle(page, timeout=5000)@@ -115,22 +117,31 @@ class StealthySession(SyncSession, StealthySessionMixin):if not challenge_type:log.error("No Cloudflare challenge found.")return None+ elif _attempts >= __CF_MAX_SOLVE_ATTEMPTS__:+ log.error(f"Failed to solve the Cloudflare challenge after {_attempts} attempts, returning the page as is")+ return Noneelse:log.info(f'The turnstile version discovered is "{challenge_type}"')if challenge_type == "non-interactive":- while "<title>Just a moment...</title>" in (ResponseFactory._get_page_content(page)):+ while self._detect_cloudflare(ResponseFactory._get_page_content(page)) == "non-interactive":log.info("Waiting for Cloudflare wait page to disappear.")page.wait_for_timeout(1000)page.wait_for_load_state()- log.info("Cloudflare captcha is solved")- return None+ if self._challenge_cleared(ResponseFactory._get_page_content(page), challenge_type):+ log.info("Cloudflare captcha is solved")+ return None+ return self._cloudflare_solver(page, _attempts + 1)else:box_selector = "#cf_turnstile div, #cf-turnstile div, .turnstile>div>div"if challenge_type != "embedded":box_selector = ".main-content p+div>div>div"- while "Verifying you are human." in ResponseFactory._get_page_content(page):- # Waiting for the verify spinner to disappear, checking every 1s if it disappeared+ for _ in range(20):+ # Waiting for the verify spinner to resolve into the widget iframe or pass on its own+ if page.frame(url=__CF_PATTERN__) is not None or self._challenge_cleared(+ ResponseFactory._get_page_content(page), challenge_type+ ):+ breakpage.wait_for_timeout(500)outer_box: Any = {}@@ -146,11 +157,14 @@ class StealthySession(SyncSession, StealthySessionMixin):outer_box = iframe.frame_element().bounding_box()if not iframe or not outer_box:- if "<title>Just a moment...</title>" not in (ResponseFactory._get_page_content(page)):+ if self._challenge_cleared(ResponseFactory._get_page_content(page), challenge_type):log.info("Cloudflare captcha is solved")return Noneouter_box = page.locator(box_selector).last.bounding_box()+ if not outer_box:+ page.wait_for_timeout(1000)+ return self._cloudflare_solver(page, _attempts + 1)# Calculate the Captcha coordinates for any viewportcaptcha_x, captcha_y = outer_box["x"] + randint(26, 28), outer_box["y"] + randint(25, 27)@@ -161,7 +175,7 @@ class StealthySession(SyncSession, StealthySessionMixin):if challenge_type != "embedded":attempts = 0- while "<title>Just a moment...</title>" in ResponseFactory._get_page_content(page):+ while not self._challenge_cleared(ResponseFactory._get_page_content(page), challenge_type):# Wait for the pageif attempts >= 100:log.info("Cloudflare page didn't disappear after 10s, continuing...")@@ -169,17 +183,14 @@ class StealthySession(SyncSession, StealthySessionMixin):page.wait_for_timeout(100)attempts += 1- # page.locator(box_selector).last.wait_for(state="detached")- # page.locator(".zone-name-title").wait_for(state="hidden")-self._wait_for_page_stability(page, True, False)- if "<title>Just a moment...</title>" not in (ResponseFactory._get_page_content(page)):+ if self._challenge_cleared(ResponseFactory._get_page_content(page), challenge_type):log.info("Cloudflare captcha is solved")return Noneelse:log.info("Looks like Cloudflare captcha is still present, solving again")- return self._cloudflare_solver(page)+ return self._cloudflare_solver(page, _attempts + 1)def fetch(self, url: str, **kwargs: Unpack[StealthFetchParams]) -> Response:"""Opens up the browser and do your request based on your chosen options.@@ -379,10 +390,11 @@ class AsyncStealthySession(AsyncSession, StealthySessionMixin):else:raise RuntimeError("Session has been already started")- async def _cloudflare_solver(self, page: async_Page) -> None: # pragma: no cover+ async def _cloudflare_solver(self, page: async_Page, _attempts: int = 0) -> None: # pragma: no cover"""Solve the cloudflare challenge displayed on the playwright page passed:param page: The targeted page+ :param _attempts: The number of solve attempts done so far, used internally to cap the retries.:return:"""await self._wait_for_networkidle(page, timeout=5000)@@ -390,22 +402,31 @@ class AsyncStealthySession(AsyncSession, StealthySessionMixin):if not challenge_type:log.error("No Cloudflare challenge found.")return None+ elif _attempts >= __CF_MAX_SOLVE_ATTEMPTS__:+ log.error(f"Failed to solve the Cloudflare challenge after {_attempts} attempts, returning the page as is")+ return Noneelse:log.info(f'The turnstile version discovered is "{challenge_type}"')if challenge_type == "non-interactive":- while "<title>Just a moment...</title>" in (await ResponseFactory._get_async_page_content(page)):+ while self._detect_cloudflare(await ResponseFactory._get_async_page_content(page)) == "non-interactive":log.info("Waiting for Cloudflare wait page to disappear.")await page.wait_for_timeout(1000)await page.wait_for_load_state()- log.info("Cloudflare captcha is solved")- return None+ if self._challenge_cleared(await ResponseFactory._get_async_page_content(page), challenge_type):+ log.info("Cloudflare captcha is solved")+ return None+ return await self._cloudflare_solver(page, _attempts + 1)else:box_selector = "#cf_turnstile div, #cf-turnstile div, .turnstile>div>div"if challenge_type != "embedded":box_selector = ".main-content p+div>div>div"- while "Verifying you are human." in (await ResponseFactory._get_async_page_content(page)):- # Waiting for the verify spinner to disappear, checking every 1s if it disappeared+ for _ in range(20):+ # Waiting for the verify spinner to resolve into the widget iframe or pass on its own+ if page.frame(url=__CF_PATTERN__) is not None or self._challenge_cleared(+ await ResponseFactory._get_async_page_content(page), challenge_type+ ):+ breakawait page.wait_for_timeout(500)outer_box: Any = {}@@ -421,11 +442,14 @@ class AsyncStealthySession(AsyncSession, StealthySessionMixin):outer_box = await (await iframe.frame_element()).bounding_box()if not iframe or not outer_box:- if "<title>Just a moment...</title>" not in (await ResponseFactory._get_async_page_content(page)):+ if self._challenge_cleared(await ResponseFactory._get_async_page_content(page), challenge_type):log.info("Cloudflare captcha is solved")return Noneouter_box = await page.locator(box_selector).last.bounding_box()+ if not outer_box:+ await page.wait_for_timeout(1000)+ return await self._cloudflare_solver(page, _attempts + 1)# Calculate the Captcha coordinates for any viewportcaptcha_x, captcha_y = outer_box["x"] + randint(26, 28), outer_box["y"] + randint(25, 27)@@ -436,7 +460,9 @@ class AsyncStealthySession(AsyncSession, StealthySessionMixin):if challenge_type != "embedded":attempts = 0- while "<title>Just a moment...</title>" in (await ResponseFactory._get_async_page_content(page)):+ while not self._challenge_cleared(+ await ResponseFactory._get_async_page_content(page), challenge_type+ ):# Wait for the pageif attempts >= 100:log.info("Cloudflare page didn't disappear after 10s, continuing...")@@ -444,17 +470,14 @@ class AsyncStealthySession(AsyncSession, StealthySessionMixin):await page.wait_for_timeout(100)attempts += 1- # await page.locator(box_selector).last.wait_for(state="detached")- # await page.locator(".zone-name-title").wait_for(state="hidden")-await self._wait_for_page_stability(page, True, False)- if "<title>Just a moment...</title>" not in (await ResponseFactory._get_async_page_content(page)):+ if self._challenge_cleared(await ResponseFactory._get_async_page_content(page), challenge_type):log.info("Cloudflare captcha is solved")return Noneelse:log.info("Looks like Cloudflare captcha is still present, solving again")- return await self._cloudflare_solver(page)+ return await self._cloudflare_solver(page, _attempts + 1)async def fetch(self, url: str, **kwargs: Unpack[StealthFetchParams]) -> Response:"""Opens up the browser and do your request based on your chosen options.diff --git a/scrapling/engines/toolbelt/convertor.py b/scrapling/engines/toolbelt/convertor.pyindex e875cf4..3552b26 100644--- a/scrapling/engines/toolbelt/convertor.py+++ b/scrapling/engines/toolbelt/convertor.py@@ -3,6 +3,7 @@ from re import compile as re_compilefrom curl_cffi.requests import Response as CurlResponsefrom playwright._impl._errors import Error as PlaywrightError+from patchright._impl._errors import Error as PatchrightErrorfrom playwright.sync_api import Page as SyncPage, Response as SyncResponsefrom playwright.async_api import Page as AsyncPage, Response as AsyncResponse@@ -206,7 +207,7 @@ class ResponseFactory:for _ in range(max_retries):try:return page.content() or ""- except PlaywrightError:+ except (PlaywrightError, PatchrightError):page.wait_for_timeout(500)raise RuntimeError(f"Failed to retrieve the page content after retrying for {max_retries * 500}ms.")@@ -221,7 +222,7 @@ class ResponseFactory:for _ in range(max_retries):try:return (await page.content()) or ""- except PlaywrightError:+ except (PlaywrightError, PatchrightError):await page.wait_for_timeout(500)raise RuntimeError(f"Failed to retrieve the page content after retrying for {max_retries * 500}ms.")diff --git a/tests/fetchers/test_cloudflare_solver.py b/tests/fetchers/test_cloudflare_solver.pynew file mode 100644index 0000000..e9915c7--- /dev/null+++ b/tests/fetchers/test_cloudflare_solver.py@@ -0,0 +1,164 @@+from unittest.mock import AsyncMock, MagicMock++import pytest+from patchright._impl._errors import Error as PatchrightError+from playwright._impl._errors import Error as PlaywrightError++from scrapling.engines.toolbelt.convertor import ResponseFactory+from scrapling.engines._browsers._stealth import StealthySession, AsyncStealthySession, __CF_MAX_SOLVE_ATTEMPTS__++FRENCH_CHALLENGE = (+ "<html><head><title>Un instant…</title></head><body><script>cType: 'interactive'</script></body></html>"+)+CLEAN_PAGE = "<html><head><title>La Redoute</title></head><body>content</body></html>"+++class TestChallengeCleared:+ def test_localized_challenge_is_not_cleared(self):+ """The check must catch the challenge markers no matter what language the page is displayed with"""+ assert StealthySession._challenge_cleared(FRENCH_CHALLENGE, "interactive") is False++ def test_clean_page_is_cleared(self):+ assert StealthySession._challenge_cleared(CLEAN_PAGE, "interactive") is True++ def test_embedded_is_always_cleared(self):+ """Embedded widgets stay in the page after solving, so they are always reported as cleared"""+ content = '<script src="https://challenges.cloudflare.com/turnstile/v0/api.js"></script>'+ assert StealthySession._challenge_cleared(content, "embedded") is True+++def _make_sync_page():+ page = MagicMock()+ page.frame.return_value = None+ page.locator.return_value.last.bounding_box.return_value = {"x": 100, "y": 200}+ return page+++class TestSyncSolver:+ def _solver_setup(self, monkeypatch, content_state):+ monkeypatch.setattr(ResponseFactory, "_get_page_content", staticmethod(lambda page: content_state["content"]))+ monkeypatch.setattr(StealthySession, "_wait_for_networkidle", lambda self, page, timeout=None: None)+ monkeypatch.setattr(+ StealthySession, "_wait_for_page_stability", lambda self, page, load_dom, network_idle: None+ )+ return object.__new__(StealthySession)++ def test_solver_clicks_localized_challenge(self, monkeypatch):+ """A localized challenge page must still be detected, clicked, and confirmed as solved"""+ state = {"content": FRENCH_CHALLENGE}+ session = self._solver_setup(monkeypatch, state)+ page = _make_sync_page()+ page.mouse.click.side_effect = lambda *args, **kwargs: state.update(content=CLEAN_PAGE)++ assert session._cloudflare_solver(page) is None+ assert page.mouse.click.call_count == 1+ x, y = page.mouse.click.call_args.args+ assert 126 <= x <= 128+ assert 225 <= y <= 227++ def test_solver_stops_after_max_attempts(self, monkeypatch):+ """A challenge that never clears must stop after the attempts cap instead of retrying forever"""+ state = {"content": FRENCH_CHALLENGE}+ session = self._solver_setup(monkeypatch, state)+ page = _make_sync_page()++ assert session._cloudflare_solver(page) is None+ assert page.mouse.click.call_count == __CF_MAX_SOLVE_ATTEMPTS__+++class TestAsyncSolver:+ def _solver_setup(self, monkeypatch, content_state):+ async def content(page):+ return content_state["content"]++ monkeypatch.setattr(ResponseFactory, "_get_async_page_content", staticmethod(content))+ monkeypatch.setattr(AsyncStealthySession, "_wait_for_networkidle", AsyncMock())+ monkeypatch.setattr(AsyncStealthySession, "_wait_for_page_stability", AsyncMock())+ return object.__new__(AsyncStealthySession)++ @pytest.mark.asyncio+ async def test_solver_clicks_localized_challenge(self, monkeypatch):+ state = {"content": FRENCH_CHALLENGE}+ session = self._solver_setup(monkeypatch, state)+ page = MagicMock()+ page.frame.return_value = None+ page.wait_for_timeout = AsyncMock()+ page.wait_for_load_state = AsyncMock()+ page.locator.return_value.last.bounding_box = AsyncMock(return_value={"x": 100, "y": 200})+ page.mouse.click = AsyncMock(side_effect=lambda *args, **kwargs: state.update(content=CLEAN_PAGE))++ assert await session._cloudflare_solver(page) is None+ assert page.mouse.click.await_count == 1+ x, y = page.mouse.click.await_args.args+ assert 126 <= x <= 128+ assert 225 <= y <= 227++ @pytest.mark.asyncio+ async def test_solver_stops_after_max_attempts(self, monkeypatch):+ state = {"content": FRENCH_CHALLENGE}+ session = self._solver_setup(monkeypatch, state)+ page = MagicMock()+ page.frame.return_value = None+ page.wait_for_timeout = AsyncMock()+ page.wait_for_load_state = AsyncMock()+ page.locator.return_value.last.bounding_box = AsyncMock(return_value={"x": 100, "y": 200})+ page.mouse.click = AsyncMock()++ assert await session._cloudflare_solver(page) is None+ assert page.mouse.click.await_count == __CF_MAX_SOLVE_ATTEMPTS__+++class TestPageContentRetries:+ @pytest.mark.parametrize("error_class", [PatchrightError, PlaywrightError])+ def test_sync_content_errors_are_retried(self, error_class):+ """Both patchright and playwright errors must trigger the retry workaround, not crash"""+ page = MagicMock()+ page.content.side_effect = [error_class("Page.content: page is navigating"), CLEAN_PAGE]++ assert ResponseFactory._get_page_content(page) == CLEAN_PAGE+ assert page.wait_for_timeout.call_count == 1++ @pytest.mark.asyncio+ @pytest.mark.parametrize("error_class", [PatchrightError, PlaywrightError])+ async def test_async_content_errors_are_retried(self, error_class):+ page = MagicMock()+ page.content = AsyncMock(side_effect=[error_class("Page.content: page is navigating"), CLEAN_PAGE])+ page.wait_for_timeout = AsyncMock()++ assert await ResponseFactory._get_async_page_content(page) == CLEAN_PAGE+ assert page.wait_for_timeout.await_count == 1++ def test_sync_content_gives_up_after_max_retries(self):+ page = MagicMock()+ page.content.side_effect = PatchrightError("Page.content: page is navigating")++ with pytest.raises(RuntimeError, match="Failed to retrieve the page content"):+ ResponseFactory._get_page_content(page, max_retries=3)+ assert page.content.call_count == 3+++class TestLocaleLaunchFlags:+ def test_locale_becomes_launch_flags(self):+ """The locale must be set browser-wide via launch flags, not the detectable context override"""+ session = StealthySession(locale="fr-FR")+ assert "--lang=fr-FR" in session._browser_options["args"]+ assert "--accept-lang=fr-FR,fr" in session._browser_options["args"]+ assert "locale" not in session._context_options++ def test_no_locale_adds_no_flags(self):+ session = StealthySession()+ assert not [f for f in session._browser_options["args"] if f.startswith(("--lang=", "--accept-lang="))]+ assert "locale" not in session._context_options++ def test_cdp_url_keeps_context_locale(self):+ """Remote browsers can't take launch flags, so the context option is the best effort left"""+ session = StealthySession(locale="fr-FR", cdp_url="ws://127.0.0.1:9222/devtools/browser/x")+ assert session._context_options.get("locale") == "fr-FR"++ def test_dynamic_session_gets_the_same_flags(self):+ from scrapling.engines._browsers._controllers import DynamicSession++ session = DynamicSession(locale="en-GB")+ assert "--lang=en-GB" in session._browser_options["args"]+ assert "--accept-lang=en-GB,en" in session._browser_options["args"]+ assert "locale" not in session._context_options