Skip to content

🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev

fix(cloudflare): Limit the waiting period for websites that captcha doesn't disappear after solving

Karim shoairfc4a809985dd
diff --git a/scrapling/engines/_browsers/_camoufox.py b/scrapling/engines/_browsers/_camoufox.py
index c0b2f2c..4ac0558 100644
--- a/scrapling/engines/_browsers/_camoufox.py
+++ b/scrapling/engines/_browsers/_camoufox.py
@@ -231,9 +231,14 @@ class StealthySession(StealthySessionMixin, SyncSession):
page.mouse.click(captcha_x, captcha_y, delay=60, button="left")
self._wait_for_networkidle(page)
if iframe is not None:
- # Wait for the frame to be removed from the page
+ # Wait for the frame to be removed from the page (with 30s timeout = 300 iterations * 100 ms)
+ attempts = 0
while iframe in page.frames:
+ if attempts >= 300:
+ log.info("Cloudflare iframe didn't disappear after 30s, continuing...")
+ break
page.wait_for_timeout(100)
+ attempts += 1
if challenge_type != "embedded":
page.locator(box_selector).last.wait_for(state="detached")
page.locator(".zone-name-title").wait_for(state="hidden")
@@ -513,9 +518,14 @@ class AsyncStealthySession(StealthySessionMixin, AsyncSession):
await page.mouse.click(captcha_x, captcha_y, delay=60, button="left")
await self._wait_for_networkidle(page)
if iframe is not None:
- # Wait for the frame to be removed from the page
+ # Wait for the frame to be removed from the page (with 30s timeout = 300 iterations * 100 ms)
+ attempts = 0
while iframe in page.frames:
+ if attempts >= 300:
+ log.info("Cloudflare iframe didn't disappear after 30s, continuing...")
+ break
await page.wait_for_timeout(100)
+ attempts += 1
if challenge_type != "embedded":
await page.locator(box_selector).wait_for(state="detached")
await page.locator(".zone-name-title").wait_for(state="hidden")