🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev
Python aiai-scrapingautomationcrawlercrawlingcrawling-pythondatadata-extractionmcpmcp-serverplaywrightpythonscrapingselectorsstealthweb-scraperweb-scrapingweb-scraping-pythonwebscrapingxpath
feat(cf solver): Make solver able to handle Turnstile and interstitial
Karim shoair716a5c357242
diff --git a/scrapling/engines/_browsers/_base.py b/scrapling/engines/_browsers/_base.pyindex 69681ee..d1a872d 100644--- a/scrapling/engines/_browsers/_base.py+++ b/scrapling/engines/_browsers/_base.py@@ -12,17 +12,13 @@ from camoufox.utils import (installed_verstr as camoufox_version,)-from scrapling.engines.toolbelt.navigation import intercept_route, async_intercept_route-from scrapling.core._types import (- Any,- Dict,- Optional,-)from ._page import PageInfo, PagePool-from ._config_tools import _compiled_stealth_scripts-from ._config_tools import _launch_kwargs, _context_kwargs+from scrapling.parser import Selector+from scrapling.core._types import Dict, Optionalfrom scrapling.engines.toolbelt.fingerprints import get_os_namefrom ._validators import validate, PlaywrightConfig, CamoufoxConfig+from ._config_tools import _compiled_stealth_scripts, _launch_kwargs, _context_kwargs+from scrapling.engines.toolbelt.navigation import intercept_route, async_intercept_route__ff_version_str__ = camoufox_version().split(".", 1)[0]@@ -268,4 +264,9 @@ class StealthySessionMixin:if f"cType: '{ctype}'" in page_content:return ctype+ # Check if turnstile captcha is embedded inside the page (Usually inside a closed Shadow iframe)+ selector = Selector(content=page_content)+ if selector.css('script[src*="challenges.cloudflare.com/turnstile/v"]'):+ return "embedded"+return Nonediff --git a/scrapling/engines/_browsers/_camoufox.py b/scrapling/engines/_browsers/_camoufox.pyindex 5f0e795..207be90 100644--- a/scrapling/engines/_browsers/_camoufox.py+++ b/scrapling/engines/_browsers/_camoufox.py@@ -237,26 +237,33 @@ class StealthySession(StealthySessionMixin, SyncSession):returnelse:- while "Verifying you are human." in self._get_page_content(page):- # Waiting for the verify spinner to disappear, checking every 1s if it disappeared- page.wait_for_timeout(500)+ box_selector = "#cf_turnstile div, #cf-turnstile div, .turnstile>div>div"+ if challenge_type != "embedded":+ box_selector = ".main-content p+div>div>div"+ while "Verifying you are human." in self._get_page_content(page):+ # Waiting for the verify spinner to disappear, checking every 1s if it disappeared+ page.wait_for_timeout(500)iframe = page.frame(url=__CF_PATTERN__)if iframe is None:- log.info("Didn't find Cloudflare iframe!")+ log.error("Didn't find Cloudflare iframe!")return- while not iframe.frame_element().is_visible():- # Double-checking that the iframe is loaded- page.wait_for_timeout(500)+ if challenge_type != "embedded":+ while not iframe.frame_element().is_visible():+ # Double-checking that the iframe is loaded+ page.wait_for_timeout(500)+ iframe.wait_for_load_state(state="domcontentloaded")+ iframe.wait_for_load_state("networkidle")# Calculate the Captcha coordinates for any viewport- outer_box = page.locator(".main-content p+div>div>div").bounding_box()+ outer_box = page.locator(box_selector).last.bounding_box()captcha_x, captcha_y = outer_box["x"] + 26, outer_box["y"] + 25# Move the mouse to the center of the window, then press and hold the left mouse buttonpage.mouse.click(captcha_x, captcha_y, delay=60, button="left")- page.locator(".zone-name-title").wait_for(state="hidden")+ if challenge_type != "embedded":+ page.locator(".zone-name-title").wait_for(state="hidden")page.wait_for_load_state(state="domcontentloaded")log.info("Cloudflare captcha is solved")@@ -556,26 +563,33 @@ class AsyncStealthySession(StealthySessionMixin, AsyncSession):returnelse:- while "Verifying you are human." in (await self._get_page_content(page)):- # Waiting for the verify spinner to disappear, checking every 1s if it disappeared- await page.wait_for_timeout(500)+ box_selector = "#cf_turnstile div, #cf-turnstile div, .turnstile>div>div"+ if challenge_type != "embedded":+ box_selector = ".main-content p+div>div>div"+ while "Verifying you are human." in (await self._get_page_content(page)):+ # Waiting for the verify spinner to disappear, checking every 1s if it disappeared+ await page.wait_for_timeout(500)iframe = page.frame(url=__CF_PATTERN__)if iframe is None:- log.info("Didn't find Cloudflare iframe!")+ log.error("Didn't find Cloudflare iframe!")return- while not await (await iframe.frame_element()).is_visible():- # Double-checking that the iframe is loaded- await page.wait_for_timeout(500)+ if challenge_type != "embedded":+ while not await (await iframe.frame_element()).is_visible():+ # Double-checking that the iframe is loaded+ await page.wait_for_timeout(500)+ await iframe.wait_for_load_state(state="domcontentloaded")+ await iframe.wait_for_load_state("networkidle")# Calculate the Captcha coordinates for any viewport- outer_box = await page.locator(".main-content p+div>div>div").bounding_box()+ outer_box = await page.locator(box_selector).last.bounding_box()captcha_x, captcha_y = outer_box["x"] + 26, outer_box["y"] + 25# Move the mouse to the center of the window, then press and hold the left mouse buttonawait page.mouse.click(captcha_x, captcha_y, delay=60, button="left")- await page.locator(".zone-name-title").wait_for(state="hidden")+ if challenge_type != "embedded":+ await page.locator(".zone-name-title").wait_for(state="hidden")await page.wait_for_load_state(state="domcontentloaded")log.info("Cloudflare captcha is solved")