🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev
Python aiai-scrapingautomationcrawlercrawlingcrawling-pythondatadata-extractionmcpmcp-serverplaywrightpythonscrapingselectorsstealthweb-scraperweb-scrapingweb-scraping-pythonwebscrapingxpath
fix(cloudflare): Limit the waiting period for websites that captcha doesn't disappear after solving
Karim shoairfc4a809985dd
diff --git a/scrapling/engines/_browsers/_camoufox.py b/scrapling/engines/_browsers/_camoufox.pyindex c0b2f2c..4ac0558 100644--- a/scrapling/engines/_browsers/_camoufox.py+++ b/scrapling/engines/_browsers/_camoufox.py@@ -231,9 +231,14 @@ class StealthySession(StealthySessionMixin, SyncSession):page.mouse.click(captcha_x, captcha_y, delay=60, button="left")self._wait_for_networkidle(page)if iframe is not None:- # Wait for the frame to be removed from the page+ # Wait for the frame to be removed from the page (with 30s timeout = 300 iterations * 100 ms)+ attempts = 0while iframe in page.frames:+ if attempts >= 300:+ log.info("Cloudflare iframe didn't disappear after 30s, continuing...")+ breakpage.wait_for_timeout(100)+ attempts += 1if challenge_type != "embedded":page.locator(box_selector).last.wait_for(state="detached")page.locator(".zone-name-title").wait_for(state="hidden")@@ -513,9 +518,14 @@ class AsyncStealthySession(StealthySessionMixin, AsyncSession):await page.mouse.click(captcha_x, captcha_y, delay=60, button="left")await self._wait_for_networkidle(page)if iframe is not None:- # Wait for the frame to be removed from the page+ # Wait for the frame to be removed from the page (with 30s timeout = 300 iterations * 100 ms)+ attempts = 0while iframe in page.frames:+ if attempts >= 300:+ log.info("Cloudflare iframe didn't disappear after 30s, continuing...")+ breakawait page.wait_for_timeout(100)+ attempts += 1if challenge_type != "embedded":await page.locator(box_selector).wait_for(state="detached")await page.locator(".zone-name-title").wait_for(state="hidden")