Skip to content

🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev

fix(stealth): solve Cloudflare challenges regardless of locale + retry cap + fix crashing mid-solve

Karim shoairc0326263b525
diff --git a/scrapling/engines/_browsers/_base.py b/scrapling/engines/_browsers/_base.py
index c9d390c..12a44c5 100644
--- a/scrapling/engines/_browsers/_base.py
+++ b/scrapling/engines/_browsers/_base.py
@@ -437,11 +437,13 @@ class BaseSessionMixin:
self._context_options.update(
{
"proxy": config.proxy,
- "locale": config.locale,
"timezone_id": config.timezone_id,
"extra_http_headers": config.extra_headers,
}
)
+ if config.locale and config.cdp_url:
+ # Launch flags can't be set on remote browsers, so the detectable context option is the best effort left
+ self._context_options["locale"] = config.locale
# The default useragent in the headful is always correct now in the current versions of Playwright
if config.useragent:
self._context_options["user_agent"] = config.useragent
@@ -462,6 +464,17 @@ class BaseSessionMixin:
else:
flags = list(flags) + [doh_flag]
+ if config.locale:
+ # The context `locale` option patches the main thread only, so Web Workers keep the browser's real
+ # language and WAFs like Cloudflare flag the mismatch. Launch flags set it browser-wide instead,
+ # so workers, `Intl`, and the `Accept-Language` header all follow natively.
+ base_lang = config.locale.split("-")[0].lower()
+ accept_lang = f"{config.locale},{base_lang}" if base_lang != config.locale.lower() else config.locale
+ flags = (flags if isinstance(flags, list) else list(flags)) + [
+ f"--lang={config.locale}",
+ f"--accept-lang={accept_lang}",
+ ]
+
self._browser_options.update(
{
"args": flags,
@@ -575,3 +588,17 @@ class StealthySessionMixin(BaseSessionMixin):
return "embedded"
return None
+
+ @classmethod
+ def _challenge_cleared(cls, page_content: str, challenge_type: str) -> bool:
+ """
+ Check whether the Cloudflare challenge is no longer present in the page content.
+
+ Args:
+ page_content (str): The content of the page to analyze.
+ challenge_type (str): The challenge type returned by `_detect_cloudflare`.
+
+ Returns:
+ bool: True if the challenge is gone from the page, False otherwise.
+ """
+ return challenge_type == "embedded" or cls._detect_cloudflare(page_content) is None
diff --git a/scrapling/engines/_browsers/_stealth.py b/scrapling/engines/_browsers/_stealth.py
index efa195e..0eb98e3 100644
--- a/scrapling/engines/_browsers/_stealth.py
+++ b/scrapling/engines/_browsers/_stealth.py
@@ -17,6 +17,7 @@ from scrapling.engines._browsers._base import SyncSession, AsyncSession, Stealth
from scrapling.engines._browsers._validators import validate_fetch as _validate, StealthConfig
__CF_PATTERN__ = re_compile(r"^https?://challenges\.cloudflare\.com/cdn-cgi/challenge-platform/.*")
+__CF_MAX_SOLVE_ATTEMPTS__ = 3
class StealthySession(SyncSession, StealthySessionMixin):
@@ -104,10 +105,11 @@ class StealthySession(SyncSession, StealthySessionMixin):
else:
raise RuntimeError("Session has been already started")
- def _cloudflare_solver(self, page: Page) -> None: # pragma: no cover
+ def _cloudflare_solver(self, page: Page, _attempts: int = 0) -> None: # pragma: no cover
"""Solve the cloudflare challenge displayed on the playwright page passed
:param page: The targeted page
+ :param _attempts: The number of solve attempts done so far, used internally to cap the retries.
:return:
"""
self._wait_for_networkidle(page, timeout=5000)
@@ -115,22 +117,31 @@ class StealthySession(SyncSession, StealthySessionMixin):
if not challenge_type:
log.error("No Cloudflare challenge found.")
return None
+ elif _attempts >= __CF_MAX_SOLVE_ATTEMPTS__:
+ log.error(f"Failed to solve the Cloudflare challenge after {_attempts} attempts, returning the page as is")
+ return None
else:
log.info(f'The turnstile version discovered is "{challenge_type}"')
if challenge_type == "non-interactive":
- while "<title>Just a moment...</title>" in (ResponseFactory._get_page_content(page)):
+ while self._detect_cloudflare(ResponseFactory._get_page_content(page)) == "non-interactive":
log.info("Waiting for Cloudflare wait page to disappear.")
page.wait_for_timeout(1000)
page.wait_for_load_state()
- log.info("Cloudflare captcha is solved")
- return None
+ if self._challenge_cleared(ResponseFactory._get_page_content(page), challenge_type):
+ log.info("Cloudflare captcha is solved")
+ return None
+ return self._cloudflare_solver(page, _attempts + 1)
else:
box_selector = "#cf_turnstile div, #cf-turnstile div, .turnstile>div>div"
if challenge_type != "embedded":
box_selector = ".main-content p+div>div>div"
- while "Verifying you are human." in ResponseFactory._get_page_content(page):
- # Waiting for the verify spinner to disappear, checking every 1s if it disappeared
+ for _ in range(20):
+ # Waiting for the verify spinner to resolve into the widget iframe or pass on its own
+ if page.frame(url=__CF_PATTERN__) is not None or self._challenge_cleared(
+ ResponseFactory._get_page_content(page), challenge_type
+ ):
+ break
page.wait_for_timeout(500)
outer_box: Any = {}
@@ -146,11 +157,14 @@ class StealthySession(SyncSession, StealthySessionMixin):
outer_box = iframe.frame_element().bounding_box()
if not iframe or not outer_box:
- if "<title>Just a moment...</title>" not in (ResponseFactory._get_page_content(page)):
+ if self._challenge_cleared(ResponseFactory._get_page_content(page), challenge_type):
log.info("Cloudflare captcha is solved")
return None
outer_box = page.locator(box_selector).last.bounding_box()
+ if not outer_box:
+ page.wait_for_timeout(1000)
+ return self._cloudflare_solver(page, _attempts + 1)
# Calculate the Captcha coordinates for any viewport
captcha_x, captcha_y = outer_box["x"] + randint(26, 28), outer_box["y"] + randint(25, 27)
@@ -161,7 +175,7 @@ class StealthySession(SyncSession, StealthySessionMixin):
if challenge_type != "embedded":
attempts = 0
- while "<title>Just a moment...</title>" in ResponseFactory._get_page_content(page):
+ while not self._challenge_cleared(ResponseFactory._get_page_content(page), challenge_type):
# Wait for the page
if attempts >= 100:
log.info("Cloudflare page didn't disappear after 10s, continuing...")
@@ -169,17 +183,14 @@ class StealthySession(SyncSession, StealthySessionMixin):
page.wait_for_timeout(100)
attempts += 1
- # page.locator(box_selector).last.wait_for(state="detached")
- # page.locator(".zone-name-title").wait_for(state="hidden")
-
self._wait_for_page_stability(page, True, False)
- if "<title>Just a moment...</title>" not in (ResponseFactory._get_page_content(page)):
+ if self._challenge_cleared(ResponseFactory._get_page_content(page), challenge_type):
log.info("Cloudflare captcha is solved")
return None
else:
log.info("Looks like Cloudflare captcha is still present, solving again")
- return self._cloudflare_solver(page)
+ return self._cloudflare_solver(page, _attempts + 1)
def fetch(self, url: str, **kwargs: Unpack[StealthFetchParams]) -> Response:
"""Opens up the browser and do your request based on your chosen options.
@@ -379,10 +390,11 @@ class AsyncStealthySession(AsyncSession, StealthySessionMixin):
else:
raise RuntimeError("Session has been already started")
- async def _cloudflare_solver(self, page: async_Page) -> None: # pragma: no cover
+ async def _cloudflare_solver(self, page: async_Page, _attempts: int = 0) -> None: # pragma: no cover
"""Solve the cloudflare challenge displayed on the playwright page passed
:param page: The targeted page
+ :param _attempts: The number of solve attempts done so far, used internally to cap the retries.
:return:
"""
await self._wait_for_networkidle(page, timeout=5000)
@@ -390,22 +402,31 @@ class AsyncStealthySession(AsyncSession, StealthySessionMixin):
if not challenge_type:
log.error("No Cloudflare challenge found.")
return None
+ elif _attempts >= __CF_MAX_SOLVE_ATTEMPTS__:
+ log.error(f"Failed to solve the Cloudflare challenge after {_attempts} attempts, returning the page as is")
+ return None
else:
log.info(f'The turnstile version discovered is "{challenge_type}"')
if challenge_type == "non-interactive":
- while "<title>Just a moment...</title>" in (await ResponseFactory._get_async_page_content(page)):
+ while self._detect_cloudflare(await ResponseFactory._get_async_page_content(page)) == "non-interactive":
log.info("Waiting for Cloudflare wait page to disappear.")
await page.wait_for_timeout(1000)
await page.wait_for_load_state()
- log.info("Cloudflare captcha is solved")
- return None
+ if self._challenge_cleared(await ResponseFactory._get_async_page_content(page), challenge_type):
+ log.info("Cloudflare captcha is solved")
+ return None
+ return await self._cloudflare_solver(page, _attempts + 1)
else:
box_selector = "#cf_turnstile div, #cf-turnstile div, .turnstile>div>div"
if challenge_type != "embedded":
box_selector = ".main-content p+div>div>div"
- while "Verifying you are human." in (await ResponseFactory._get_async_page_content(page)):
- # Waiting for the verify spinner to disappear, checking every 1s if it disappeared
+ for _ in range(20):
+ # Waiting for the verify spinner to resolve into the widget iframe or pass on its own
+ if page.frame(url=__CF_PATTERN__) is not None or self._challenge_cleared(
+ await ResponseFactory._get_async_page_content(page), challenge_type
+ ):
+ break
await page.wait_for_timeout(500)
outer_box: Any = {}
@@ -421,11 +442,14 @@ class AsyncStealthySession(AsyncSession, StealthySessionMixin):
outer_box = await (await iframe.frame_element()).bounding_box()
if not iframe or not outer_box:
- if "<title>Just a moment...</title>" not in (await ResponseFactory._get_async_page_content(page)):
+ if self._challenge_cleared(await ResponseFactory._get_async_page_content(page), challenge_type):
log.info("Cloudflare captcha is solved")
return None
outer_box = await page.locator(box_selector).last.bounding_box()
+ if not outer_box:
+ await page.wait_for_timeout(1000)
+ return await self._cloudflare_solver(page, _attempts + 1)
# Calculate the Captcha coordinates for any viewport
captcha_x, captcha_y = outer_box["x"] + randint(26, 28), outer_box["y"] + randint(25, 27)
@@ -436,7 +460,9 @@ class AsyncStealthySession(AsyncSession, StealthySessionMixin):
if challenge_type != "embedded":
attempts = 0
- while "<title>Just a moment...</title>" in (await ResponseFactory._get_async_page_content(page)):
+ while not self._challenge_cleared(
+ await ResponseFactory._get_async_page_content(page), challenge_type
+ ):
# Wait for the page
if attempts >= 100:
log.info("Cloudflare page didn't disappear after 10s, continuing...")
@@ -444,17 +470,14 @@ class AsyncStealthySession(AsyncSession, StealthySessionMixin):
await page.wait_for_timeout(100)
attempts += 1
- # await page.locator(box_selector).last.wait_for(state="detached")
- # await page.locator(".zone-name-title").wait_for(state="hidden")
-
await self._wait_for_page_stability(page, True, False)
- if "<title>Just a moment...</title>" not in (await ResponseFactory._get_async_page_content(page)):
+ if self._challenge_cleared(await ResponseFactory._get_async_page_content(page), challenge_type):
log.info("Cloudflare captcha is solved")
return None
else:
log.info("Looks like Cloudflare captcha is still present, solving again")
- return await self._cloudflare_solver(page)
+ return await self._cloudflare_solver(page, _attempts + 1)
async def fetch(self, url: str, **kwargs: Unpack[StealthFetchParams]) -> Response:
"""Opens up the browser and do your request based on your chosen options.
diff --git a/scrapling/engines/toolbelt/convertor.py b/scrapling/engines/toolbelt/convertor.py
index e875cf4..3552b26 100644
--- a/scrapling/engines/toolbelt/convertor.py
+++ b/scrapling/engines/toolbelt/convertor.py
@@ -3,6 +3,7 @@ from re import compile as re_compile
from curl_cffi.requests import Response as CurlResponse
from playwright._impl._errors import Error as PlaywrightError
+from patchright._impl._errors import Error as PatchrightError
from playwright.sync_api import Page as SyncPage, Response as SyncResponse
from playwright.async_api import Page as AsyncPage, Response as AsyncResponse
@@ -206,7 +207,7 @@ class ResponseFactory:
for _ in range(max_retries):
try:
return page.content() or ""
- except PlaywrightError:
+ except (PlaywrightError, PatchrightError):
page.wait_for_timeout(500)
raise RuntimeError(f"Failed to retrieve the page content after retrying for {max_retries * 500}ms.")
@@ -221,7 +222,7 @@ class ResponseFactory:
for _ in range(max_retries):
try:
return (await page.content()) or ""
- except PlaywrightError:
+ except (PlaywrightError, PatchrightError):
await page.wait_for_timeout(500)
raise RuntimeError(f"Failed to retrieve the page content after retrying for {max_retries * 500}ms.")
diff --git a/tests/fetchers/test_cloudflare_solver.py b/tests/fetchers/test_cloudflare_solver.py
new file mode 100644
index 0000000..e9915c7
--- /dev/null
+++ b/tests/fetchers/test_cloudflare_solver.py
@@ -0,0 +1,164 @@
+from unittest.mock import AsyncMock, MagicMock
+
+import pytest
+from patchright._impl._errors import Error as PatchrightError
+from playwright._impl._errors import Error as PlaywrightError
+
+from scrapling.engines.toolbelt.convertor import ResponseFactory
+from scrapling.engines._browsers._stealth import StealthySession, AsyncStealthySession, __CF_MAX_SOLVE_ATTEMPTS__
+
+FRENCH_CHALLENGE = (
+ "<html><head><title>Un instant…</title></head><body><script>cType: 'interactive'</script></body></html>"
+)
+CLEAN_PAGE = "<html><head><title>La Redoute</title></head><body>content</body></html>"
+
+
+class TestChallengeCleared:
+ def test_localized_challenge_is_not_cleared(self):
+ """The check must catch the challenge markers no matter what language the page is displayed with"""
+ assert StealthySession._challenge_cleared(FRENCH_CHALLENGE, "interactive") is False
+
+ def test_clean_page_is_cleared(self):
+ assert StealthySession._challenge_cleared(CLEAN_PAGE, "interactive") is True
+
+ def test_embedded_is_always_cleared(self):
+ """Embedded widgets stay in the page after solving, so they are always reported as cleared"""
+ content = '<script src="https://challenges.cloudflare.com/turnstile/v0/api.js"></script>'
+ assert StealthySession._challenge_cleared(content, "embedded") is True
+
+
+def _make_sync_page():
+ page = MagicMock()
+ page.frame.return_value = None
+ page.locator.return_value.last.bounding_box.return_value = {"x": 100, "y": 200}
+ return page
+
+
+class TestSyncSolver:
+ def _solver_setup(self, monkeypatch, content_state):
+ monkeypatch.setattr(ResponseFactory, "_get_page_content", staticmethod(lambda page: content_state["content"]))
+ monkeypatch.setattr(StealthySession, "_wait_for_networkidle", lambda self, page, timeout=None: None)
+ monkeypatch.setattr(
+ StealthySession, "_wait_for_page_stability", lambda self, page, load_dom, network_idle: None
+ )
+ return object.__new__(StealthySession)
+
+ def test_solver_clicks_localized_challenge(self, monkeypatch):
+ """A localized challenge page must still be detected, clicked, and confirmed as solved"""
+ state = {"content": FRENCH_CHALLENGE}
+ session = self._solver_setup(monkeypatch, state)
+ page = _make_sync_page()
+ page.mouse.click.side_effect = lambda *args, **kwargs: state.update(content=CLEAN_PAGE)
+
+ assert session._cloudflare_solver(page) is None
+ assert page.mouse.click.call_count == 1
+ x, y = page.mouse.click.call_args.args
+ assert 126 <= x <= 128
+ assert 225 <= y <= 227
+
+ def test_solver_stops_after_max_attempts(self, monkeypatch):
+ """A challenge that never clears must stop after the attempts cap instead of retrying forever"""
+ state = {"content": FRENCH_CHALLENGE}
+ session = self._solver_setup(monkeypatch, state)
+ page = _make_sync_page()
+
+ assert session._cloudflare_solver(page) is None
+ assert page.mouse.click.call_count == __CF_MAX_SOLVE_ATTEMPTS__
+
+
+class TestAsyncSolver:
+ def _solver_setup(self, monkeypatch, content_state):
+ async def content(page):
+ return content_state["content"]
+
+ monkeypatch.setattr(ResponseFactory, "_get_async_page_content", staticmethod(content))
+ monkeypatch.setattr(AsyncStealthySession, "_wait_for_networkidle", AsyncMock())
+ monkeypatch.setattr(AsyncStealthySession, "_wait_for_page_stability", AsyncMock())
+ return object.__new__(AsyncStealthySession)
+
+ @pytest.mark.asyncio
+ async def test_solver_clicks_localized_challenge(self, monkeypatch):
+ state = {"content": FRENCH_CHALLENGE}
+ session = self._solver_setup(monkeypatch, state)
+ page = MagicMock()
+ page.frame.return_value = None
+ page.wait_for_timeout = AsyncMock()
+ page.wait_for_load_state = AsyncMock()
+ page.locator.return_value.last.bounding_box = AsyncMock(return_value={"x": 100, "y": 200})
+ page.mouse.click = AsyncMock(side_effect=lambda *args, **kwargs: state.update(content=CLEAN_PAGE))
+
+ assert await session._cloudflare_solver(page) is None
+ assert page.mouse.click.await_count == 1
+ x, y = page.mouse.click.await_args.args
+ assert 126 <= x <= 128
+ assert 225 <= y <= 227
+
+ @pytest.mark.asyncio
+ async def test_solver_stops_after_max_attempts(self, monkeypatch):
+ state = {"content": FRENCH_CHALLENGE}
+ session = self._solver_setup(monkeypatch, state)
+ page = MagicMock()
+ page.frame.return_value = None
+ page.wait_for_timeout = AsyncMock()
+ page.wait_for_load_state = AsyncMock()
+ page.locator.return_value.last.bounding_box = AsyncMock(return_value={"x": 100, "y": 200})
+ page.mouse.click = AsyncMock()
+
+ assert await session._cloudflare_solver(page) is None
+ assert page.mouse.click.await_count == __CF_MAX_SOLVE_ATTEMPTS__
+
+
+class TestPageContentRetries:
+ @pytest.mark.parametrize("error_class", [PatchrightError, PlaywrightError])
+ def test_sync_content_errors_are_retried(self, error_class):
+ """Both patchright and playwright errors must trigger the retry workaround, not crash"""
+ page = MagicMock()
+ page.content.side_effect = [error_class("Page.content: page is navigating"), CLEAN_PAGE]
+
+ assert ResponseFactory._get_page_content(page) == CLEAN_PAGE
+ assert page.wait_for_timeout.call_count == 1
+
+ @pytest.mark.asyncio
+ @pytest.mark.parametrize("error_class", [PatchrightError, PlaywrightError])
+ async def test_async_content_errors_are_retried(self, error_class):
+ page = MagicMock()
+ page.content = AsyncMock(side_effect=[error_class("Page.content: page is navigating"), CLEAN_PAGE])
+ page.wait_for_timeout = AsyncMock()
+
+ assert await ResponseFactory._get_async_page_content(page) == CLEAN_PAGE
+ assert page.wait_for_timeout.await_count == 1
+
+ def test_sync_content_gives_up_after_max_retries(self):
+ page = MagicMock()
+ page.content.side_effect = PatchrightError("Page.content: page is navigating")
+
+ with pytest.raises(RuntimeError, match="Failed to retrieve the page content"):
+ ResponseFactory._get_page_content(page, max_retries=3)
+ assert page.content.call_count == 3
+
+
+class TestLocaleLaunchFlags:
+ def test_locale_becomes_launch_flags(self):
+ """The locale must be set browser-wide via launch flags, not the detectable context override"""
+ session = StealthySession(locale="fr-FR")
+ assert "--lang=fr-FR" in session._browser_options["args"]
+ assert "--accept-lang=fr-FR,fr" in session._browser_options["args"]
+ assert "locale" not in session._context_options
+
+ def test_no_locale_adds_no_flags(self):
+ session = StealthySession()
+ assert not [f for f in session._browser_options["args"] if f.startswith(("--lang=", "--accept-lang="))]
+ assert "locale" not in session._context_options
+
+ def test_cdp_url_keeps_context_locale(self):
+ """Remote browsers can't take launch flags, so the context option is the best effort left"""
+ session = StealthySession(locale="fr-FR", cdp_url="ws://127.0.0.1:9222/devtools/browser/x")
+ assert session._context_options.get("locale") == "fr-FR"
+
+ def test_dynamic_session_gets_the_same_flags(self):
+ from scrapling.engines._browsers._controllers import DynamicSession
+
+ session = DynamicSession(locale="en-GB")
+ assert "--lang=en-GB" in session._browser_options["args"]
+ assert "--accept-lang=en-GB,en" in session._browser_options["args"]
+ assert "locale" not in session._context_options