🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev
Python aiai-scrapingautomationcrawlercrawlingcrawling-pythondatadata-extractionmcpmcp-serverplaywrightpythonscrapingselectorsstealthweb-scraperweb-scrapingweb-scraping-pythonwebscrapingxpath
fix(StealthyFetcher): Adjustments
Karim shoairbd47c070aa74
diff --git a/scrapling/engines/camo.py b/scrapling/engines/camo.pyindex 72e33b5..98e0747 100644--- a/scrapling/engines/camo.py+++ b/scrapling/engines/camo.py@@ -227,6 +227,23 @@ class CamoufoxEngine:@staticmethoddef __detect_cloudflare(page_content):+ """+ Detect the type of Cloudflare challenge present in the provided page content.++ This function analyzes the given page content to identify whether a specific+ type of Cloudflare challenge is present. It checks for three predefined+ challenge types: non-interactive, managed, and interactive. If a challenge+ type is detected, it returns the corresponding type as a string. If no+ challenge type is detected, it returns None.++ Args:+ page_content (str): The content of the page to analyze for Cloudflare+ challenge types.++ Returns:+ str: A string representing the detected Cloudflare challenge type, if+ found. Returns None if no challenge matches.+ """challenge_types = ("non-interactive","managed",@@ -262,7 +279,7 @@ class CamoufoxEngine:else:while "Verifying you are human." in page.content():# Waiting for the verify spinner to disappear, checking every 1s if it disappeared- page.wait_for_timeout(1000)+ page.wait_for_timeout(500)iframe = page.frame(url=re.compile(@@ -270,12 +287,12 @@ class CamoufoxEngine:))if iframe is None:- print("No iframe bro")+ log.info("Didn't find Cloudflare iframe!")returnwhile not iframe.frame_element().is_visible():# Double-checking that the iframe is loaded- page.wait_for_timeout(1000)+ page.wait_for_timeout(500)# Calculate the Captcha coordinates for any viewportouter_box = page.locator(".main-content p+div>div>div").bounding_box()@@ -313,7 +330,7 @@ class CamoufoxEngine:else:while "Verifying you are human." in (await page.content()):# Waiting for the verify spinner to disappear, checking every 1s if it disappeared- await page.wait_for_timeout(1000)+ await page.wait_for_timeout(500)iframe = page.frame(url=re.compile(@@ -321,12 +338,12 @@ class CamoufoxEngine:))if iframe is None:- print("No iframe bro")+ log.info("Didn't find Cloudflare iframe!")returnwhile not await (await iframe.frame_element()).is_visible():# Double-checking that the iframe is loaded- await page.wait_for_timeout(1000)+ await page.wait_for_timeout(500)# Calculate the Captcha coordinates for any viewportouter_box = await page.locator(