Skip to content

🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev

fix(fetchers): better handling for infinite `network_idle`

Karim shoairaed34e43b5eb
diff --git a/scrapling/engines/_browsers/_base.py b/scrapling/engines/_browsers/_base.py
index f329fa4..6e5734e 100644
--- a/scrapling/engines/_browsers/_base.py
+++ b/scrapling/engines/_browsers/_base.py
@@ -105,7 +105,7 @@ class SyncSession:
"""Wait for the page to become idle (no network activity) even if there are never-ending requests."""
try:
page.wait_for_load_state("networkidle", timeout=timeout)
- except PlaywrightError:
+ except (PlaywrightError, Exception):
pass
def _wait_for_page_stability(self, page: Page | Frame, load_dom: bool, network_idle: bool):
@@ -221,7 +221,7 @@ class AsyncSession:
"""Wait for the page to become idle (no network activity) even if there are never-ending requests."""
try:
await page.wait_for_load_state("networkidle", timeout=timeout)
- except PlaywrightError:
+ except (PlaywrightError, Exception):
pass
async def _wait_for_page_stability(self, page: AsyncPage | AsyncFrame, load_dom: bool, network_idle: bool):