Skip to content

🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev

feat(cf solver): Make solver able to handle Turnstile and interstitial

Karim shoair716a5c357242
diff --git a/scrapling/engines/_browsers/_base.py b/scrapling/engines/_browsers/_base.py
index 69681ee..d1a872d 100644
--- a/scrapling/engines/_browsers/_base.py
+++ b/scrapling/engines/_browsers/_base.py
@@ -12,17 +12,13 @@ from camoufox.utils import (
installed_verstr as camoufox_version,
)
-from scrapling.engines.toolbelt.navigation import intercept_route, async_intercept_route
-from scrapling.core._types import (
- Any,
- Dict,
- Optional,
-)
from ._page import PageInfo, PagePool
-from ._config_tools import _compiled_stealth_scripts
-from ._config_tools import _launch_kwargs, _context_kwargs
+from scrapling.parser import Selector
+from scrapling.core._types import Dict, Optional
from scrapling.engines.toolbelt.fingerprints import get_os_name
from ._validators import validate, PlaywrightConfig, CamoufoxConfig
+from ._config_tools import _compiled_stealth_scripts, _launch_kwargs, _context_kwargs
+from scrapling.engines.toolbelt.navigation import intercept_route, async_intercept_route
__ff_version_str__ = camoufox_version().split(".", 1)[0]
@@ -268,4 +264,9 @@ class StealthySessionMixin:
if f"cType: '{ctype}'" in page_content:
return ctype
+ # Check if turnstile captcha is embedded inside the page (Usually inside a closed Shadow iframe)
+ selector = Selector(content=page_content)
+ if selector.css('script[src*="challenges.cloudflare.com/turnstile/v"]'):
+ return "embedded"
+
return None
diff --git a/scrapling/engines/_browsers/_camoufox.py b/scrapling/engines/_browsers/_camoufox.py
index 5f0e795..207be90 100644
--- a/scrapling/engines/_browsers/_camoufox.py
+++ b/scrapling/engines/_browsers/_camoufox.py
@@ -237,26 +237,33 @@ class StealthySession(StealthySessionMixin, SyncSession):
return
else:
- while "Verifying you are human." in self._get_page_content(page):
- # Waiting for the verify spinner to disappear, checking every 1s if it disappeared
- page.wait_for_timeout(500)
+ box_selector = "#cf_turnstile div, #cf-turnstile div, .turnstile>div>div"
+ if challenge_type != "embedded":
+ box_selector = ".main-content p+div>div>div"
+ while "Verifying you are human." in self._get_page_content(page):
+ # Waiting for the verify spinner to disappear, checking every 1s if it disappeared
+ page.wait_for_timeout(500)
iframe = page.frame(url=__CF_PATTERN__)
if iframe is None:
- log.info("Didn't find Cloudflare iframe!")
+ log.error("Didn't find Cloudflare iframe!")
return
- while not iframe.frame_element().is_visible():
- # Double-checking that the iframe is loaded
- page.wait_for_timeout(500)
+ if challenge_type != "embedded":
+ while not iframe.frame_element().is_visible():
+ # Double-checking that the iframe is loaded
+ page.wait_for_timeout(500)
+ iframe.wait_for_load_state(state="domcontentloaded")
+ iframe.wait_for_load_state("networkidle")
# Calculate the Captcha coordinates for any viewport
- outer_box = page.locator(".main-content p+div>div>div").bounding_box()
+ outer_box = page.locator(box_selector).last.bounding_box()
captcha_x, captcha_y = outer_box["x"] + 26, outer_box["y"] + 25
# Move the mouse to the center of the window, then press and hold the left mouse button
page.mouse.click(captcha_x, captcha_y, delay=60, button="left")
- page.locator(".zone-name-title").wait_for(state="hidden")
+ if challenge_type != "embedded":
+ page.locator(".zone-name-title").wait_for(state="hidden")
page.wait_for_load_state(state="domcontentloaded")
log.info("Cloudflare captcha is solved")
@@ -556,26 +563,33 @@ class AsyncStealthySession(StealthySessionMixin, AsyncSession):
return
else:
- while "Verifying you are human." in (await self._get_page_content(page)):
- # Waiting for the verify spinner to disappear, checking every 1s if it disappeared
- await page.wait_for_timeout(500)
+ box_selector = "#cf_turnstile div, #cf-turnstile div, .turnstile>div>div"
+ if challenge_type != "embedded":
+ box_selector = ".main-content p+div>div>div"
+ while "Verifying you are human." in (await self._get_page_content(page)):
+ # Waiting for the verify spinner to disappear, checking every 1s if it disappeared
+ await page.wait_for_timeout(500)
iframe = page.frame(url=__CF_PATTERN__)
if iframe is None:
- log.info("Didn't find Cloudflare iframe!")
+ log.error("Didn't find Cloudflare iframe!")
return
- while not await (await iframe.frame_element()).is_visible():
- # Double-checking that the iframe is loaded
- await page.wait_for_timeout(500)
+ if challenge_type != "embedded":
+ while not await (await iframe.frame_element()).is_visible():
+ # Double-checking that the iframe is loaded
+ await page.wait_for_timeout(500)
+ await iframe.wait_for_load_state(state="domcontentloaded")
+ await iframe.wait_for_load_state("networkidle")
# Calculate the Captcha coordinates for any viewport
- outer_box = await page.locator(".main-content p+div>div>div").bounding_box()
+ outer_box = await page.locator(box_selector).last.bounding_box()
captcha_x, captcha_y = outer_box["x"] + 26, outer_box["y"] + 25
# Move the mouse to the center of the window, then press and hold the left mouse button
await page.mouse.click(captcha_x, captcha_y, delay=60, button="left")
- await page.locator(".zone-name-title").wait_for(state="hidden")
+ if challenge_type != "embedded":
+ await page.locator(".zone-name-title").wait_for(state="hidden")
await page.wait_for_load_state(state="domcontentloaded")
log.info("Cloudflare captcha is solved")