Skip to content

🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev

feat(CF solver): better implementation to help with all custom implementations of turnstile

Karim shoaire149c715dda1
diff --git a/scrapling/engines/_browsers/_camoufox.py b/scrapling/engines/_browsers/_camoufox.py
index 519ae83..6347a6c 100644
--- a/scrapling/engines/_browsers/_camoufox.py
+++ b/scrapling/engines/_browsers/_camoufox.py
@@ -1,3 +1,4 @@
+from random import randint
from re import compile as re_compile
from playwright.sync_api import (
@@ -20,6 +21,7 @@ from ._validators import validate_fetch as _validate
from ._base import SyncSession, AsyncSession, StealthySessionMixin
from scrapling.core.utils import log
from scrapling.core._types import (
+ Any,
Dict,
List,
Optional,
@@ -33,7 +35,7 @@ from scrapling.engines.toolbelt.convertor import (
from scrapling.engines.toolbelt.fingerprints import generate_convincing_referer
__CF_PATTERN__ = re_compile("challenges.cloudflare.com/cdn-cgi/challenge-platform/.*")
-_UNSET = object()
+_UNSET: Any = object()
class StealthySession(StealthySessionMixin, SyncSession):
@@ -222,6 +224,7 @@ class StealthySession(StealthySessionMixin, SyncSession):
:param page: The targeted page
:return:
"""
+ page.wait_for_load_state("networkidle")
challenge_type = self._detect_cloudflare(self._get_page_content(page))
if not challenge_type:
log.error("No Cloudflare challenge found.")
@@ -244,25 +247,29 @@ class StealthySession(StealthySessionMixin, SyncSession):
# Waiting for the verify spinner to disappear, checking every 1s if it disappeared
page.wait_for_timeout(500)
+ outer_box = {}
iframe = page.frame(url=__CF_PATTERN__)
- if iframe is None:
- log.error("Didn't find Cloudflare iframe!")
- return
+ if iframe is not None:
+ iframe.wait_for_load_state(state="domcontentloaded")
+ iframe.wait_for_load_state("networkidle")
- if challenge_type != "embedded":
- while not iframe.frame_element().is_visible():
- # Double-checking that the iframe is loaded
- page.wait_for_timeout(500)
+ if challenge_type != "embedded":
+ while not iframe.frame_element().is_visible():
+ # Double-checking that the iframe is loaded
+ page.wait_for_timeout(500)
+ outer_box: Any = iframe.frame_element().bounding_box()
+
+ if not iframe or not outer_box:
+ outer_box: Any = page.locator(box_selector).last.bounding_box()
- iframe.wait_for_load_state(state="domcontentloaded")
- iframe.wait_for_load_state("networkidle")
# Calculate the Captcha coordinates for any viewport
- outer_box = page.locator(box_selector).last.bounding_box()
- captcha_x, captcha_y = outer_box["x"] + 26, outer_box["y"] + 25
+ captcha_x, captcha_y = outer_box["x"] + randint(26, 28), outer_box["y"] + randint(25, 27)
# Move the mouse to the center of the window, then press and hold the left mouse button
page.mouse.click(captcha_x, captcha_y, delay=60, button="left")
+ page.wait_for_load_state("networkidle")
if challenge_type != "embedded":
+ page.locator(box_selector).wait_for(state="detached")
page.locator(".zone-name-title").wait_for(state="hidden")
page.wait_for_load_state(state="domcontentloaded")
@@ -548,6 +555,7 @@ class AsyncStealthySession(StealthySessionMixin, AsyncSession):
:param page: The async targeted page
:return:
"""
+ await page.wait_for_load_state("networkidle")
challenge_type = self._detect_cloudflare(await self._get_page_content(page))
if not challenge_type:
log.error("No Cloudflare challenge found.")
@@ -570,25 +578,29 @@ class AsyncStealthySession(StealthySessionMixin, AsyncSession):
# Waiting for the verify spinner to disappear, checking every 1s if it disappeared
await page.wait_for_timeout(500)
+ outer_box = {}
iframe = page.frame(url=__CF_PATTERN__)
- if iframe is None:
- log.error("Didn't find Cloudflare iframe!")
- return
+ if iframe is not None:
+ await iframe.wait_for_load_state(state="domcontentloaded")
+ await iframe.wait_for_load_state("networkidle")
- if challenge_type != "embedded":
- while not await (await iframe.frame_element()).is_visible():
- # Double-checking that the iframe is loaded
- await page.wait_for_timeout(500)
+ if challenge_type != "embedded":
+ while not await (await iframe.frame_element()).is_visible():
+ # Double-checking that the iframe is loaded
+ await page.wait_for_timeout(500)
+ outer_box: Any = await (await iframe.frame_element()).bounding_box()
+
+ elif not iframe or not outer_box:
+ outer_box: Any = await page.locator(box_selector).last.bounding_box()
- await iframe.wait_for_load_state(state="domcontentloaded")
- await iframe.wait_for_load_state("networkidle")
# Calculate the Captcha coordinates for any viewport
- outer_box = await page.locator(box_selector).last.bounding_box()
- captcha_x, captcha_y = outer_box["x"] + 26, outer_box["y"] + 25
+ captcha_x, captcha_y = outer_box["x"] + randint(26, 28), outer_box["y"] + randint(25, 27)
# Move the mouse to the center of the window, then press and hold the left mouse button
await page.mouse.click(captcha_x, captcha_y, delay=60, button="left")
+ await page.wait_for_load_state("networkidle")
if challenge_type != "embedded":
+ await page.locator(box_selector).wait_for(state="detached")
await page.locator(".zone-name-title").wait_for(state="hidden")
await page.wait_for_load_state(state="domcontentloaded")