🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev
Python aiai-scrapingautomationcrawlercrawlingcrawling-pythondatadata-extractionmcpmcp-serverplaywrightpythonscrapingselectorsstealthweb-scraperweb-scrapingweb-scraping-pythonwebscrapingxpath
fix(CF solver): further improve the detection
Karim shoair8c995c936d50
diff --git a/scrapling/engines/_browsers/_camoufox.py b/scrapling/engines/_browsers/_camoufox.pyindex d610c3d..6e2b620 100644--- a/scrapling/engines/_browsers/_camoufox.py+++ b/scrapling/engines/_browsers/_camoufox.py@@ -208,7 +208,7 @@ class StealthySession(StealthySessionMixin, SyncSession):@staticmethoddef _get_page_content(page: Page) -> str:"""- A workaround for Playwright issue with `page.content()` on Windows. Ref.: https://github.com/microsoft/playwright/issues/16108+ A workaround for the Playwright issue with `page.content()` on Windows. Ref.: https://github.com/microsoft/playwright/issues/16108:param page: The page to extract content from.:return:"""@@ -270,8 +270,12 @@ class StealthySession(StealthySessionMixin, SyncSession):# Move the mouse to the center of the window, then press and hold the left mouse buttonpage.mouse.click(captcha_x, captcha_y, delay=60, button="left")page.wait_for_load_state("networkidle")+ if iframe is not None:+ # Wait for the frame to be removed from the page+ while iframe in page.frames:+ page.wait_for_timeout(100)if challenge_type != "embedded":- page.locator(box_selector).wait_for(state="detached")+ page.locator(box_selector).last.wait_for(state="detached")page.locator(".zone-name-title").wait_for(state="hidden")page.wait_for_load_state(state="domcontentloaded")@@ -397,7 +401,7 @@ class StealthySession(StealthySessionMixin, SyncSession):page_info.page, first_response, final_response, params.selector_config)- # Close the page, to free up resources+ # Close the page to free up resourcespage_info.page.close()self.page_pool.pages.remove(page_info)@@ -541,7 +545,7 @@ class AsyncStealthySession(StealthySessionMixin, AsyncSession):@staticmethodasync def _get_page_content(page: async_Page) -> str:"""- A workaround for Playwright issue with `page.content()` on Windows. Ref.: https://github.com/microsoft/playwright/issues/16108+ A workaround for the Playwright issue with `page.content()` on Windows. Ref.: https://github.com/microsoft/playwright/issues/16108:param page: The page to extract content from.:return:"""@@ -603,6 +607,10 @@ class AsyncStealthySession(StealthySessionMixin, AsyncSession):# Move the mouse to the center of the window, then press and hold the left mouse buttonawait page.mouse.click(captcha_x, captcha_y, delay=60, button="left")await page.wait_for_load_state("networkidle")+ if iframe is not None:+ # Wait for the frame to be removed from the page+ while iframe in page.frames:+ await page.wait_for_timeout(100)if challenge_type != "embedded":await page.locator(box_selector).wait_for(state="detached")await page.locator(".zone-name-title").wait_for(state="hidden")@@ -736,7 +744,7 @@ class AsyncStealthySession(StealthySessionMixin, AsyncSession):page_info.page, first_response, final_response, params.selector_config)- # Close the page, to free up resources+ # Close the page to free up resourcesawait page_info.page.close()self.page_pool.pages.remove(page_info)