Skip to content

🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev

feat(browsers): Add option to retry tabs that gives errors

Karim shoaira17bb1d97607
diff --git a/scrapling/engines/_browsers/_base.py b/scrapling/engines/_browsers/_base.py
index cbbdc24..810c51c 100644
--- a/scrapling/engines/_browsers/_base.py
+++ b/scrapling/engines/_browsers/_base.py
@@ -310,6 +310,14 @@ class BaseSessionMixin:
if config.additional_args:
self._context_options.update(config.additional_args)
+ @staticmethod
+ def _is_retriable(error: Exception) -> bool:
+ """Check if an error is retriable (transient network/timeout issues)."""
+ if isinstance(error, TimeoutError):
+ return True
+ error_msg = str(error).lower()
+ return "net::" in error_msg or "failed to get response" in error_msg
+
class DynamicSessionMixin(BaseSessionMixin):
def __validate__(self, **params):
diff --git a/scrapling/engines/_browsers/_controllers.py b/scrapling/engines/_browsers/_controllers.py
index 3d39e9d..bd5d455 100644
--- a/scrapling/engines/_browsers/_controllers.py
+++ b/scrapling/engines/_browsers/_controllers.py
@@ -1,3 +1,6 @@
+from time import sleep as time_sleep
+from asyncio import sleep as asyncio_sleep
+
from playwright.sync_api import (
Locator,
Playwright,
@@ -115,52 +118,56 @@ class DynamicSession(SyncSession, DynamicSessionMixin):
else None
)
- page_info = self._get_page(params.timeout, params.extra_headers, params.disable_resources)
- final_response = [None]
- handle_response = self._create_response_handler(page_info, final_response)
-
- try: # pragma: no cover
- # Navigate to URL and wait for a specified state
- page_info.page.on("response", handle_response)
- first_response = page_info.page.goto(url, referer=referer)
- self._wait_for_page_stability(page_info.page, params.load_dom, params.network_idle)
-
- if not first_response:
- raise RuntimeError(f"Failed to get response for {url}")
-
- if params.page_action:
- try:
- _ = params.page_action(page_info.page)
- except Exception as e: # pragma: no cover
- log.error(f"Error executing page_action: {e}")
-
- if params.wait_selector:
- try:
- waiter: Locator = page_info.page.locator(params.wait_selector)
- waiter.first.wait_for(state=params.wait_selector_state)
- # Wait again after waiting for the selector, helpful with protections like Cloudflare
- self._wait_for_page_stability(page_info.page, params.load_dom, params.network_idle)
- except Exception as e: # pragma: no cover
- log.error(f"Error waiting for selector {params.wait_selector}: {e}")
-
- page_info.page.wait_for_timeout(params.wait)
+ for attempt in range(self._config.retries):
+ page_info = self._get_page(params.timeout, params.extra_headers, params.disable_resources)
+ final_response = [None]
+ handle_response = self._create_response_handler(page_info, final_response)
+
+ try: # pragma: no cover
+ page_info.page.on("response", handle_response)
+ first_response = page_info.page.goto(url, referer=referer)
+ self._wait_for_page_stability(page_info.page, params.load_dom, params.network_idle)
+
+ if not first_response:
+ raise RuntimeError(f"Failed to get response for {url}")
+
+ if params.page_action:
+ try:
+ _ = params.page_action(page_info.page)
+ except Exception as e: # pragma: no cover
+ log.error(f"Error executing page_action: {e}")
+
+ if params.wait_selector:
+ try:
+ waiter: Locator = page_info.page.locator(params.wait_selector)
+ waiter.first.wait_for(state=params.wait_selector_state)
+ self._wait_for_page_stability(page_info.page, params.load_dom, params.network_idle)
+ except Exception as e: # pragma: no cover
+ log.error(f"Error waiting for selector {params.wait_selector}: {e}")
+
+ page_info.page.wait_for_timeout(params.wait)
+
+ response = ResponseFactory.from_playwright_response(
+ page_info.page, first_response, final_response[0], params.selector_config
+ )
- # Create response object
- response = ResponseFactory.from_playwright_response(
- page_info.page, first_response, final_response[0], params.selector_config
- )
+ page_info.page.close()
+ self.page_pool.pages.remove(page_info)
+ return response
- # Close the page to free up resources
- page_info.page.close()
- self.page_pool.pages.remove(page_info)
+ except Exception as e:
+ page_info.mark_error()
+ page_info.page.close()
+ self.page_pool.pages.remove(page_info)
- return response
+ if attempt < self._config.retries - 1 and self._is_retriable(e):
+ log.warning(f"Attempt {attempt + 1} failed: {e}. Retrying in {self._config.retry_delay}s...")
+ time_sleep(self._config.retry_delay)
+ else:
+ raise
- except Exception as e:
- page_info.mark_error()
- page_info.page.close()
- self.page_pool.pages.remove(page_info)
- raise e
+ # For type checking purposes only
+ raise AssertionError("Unreachable: retry loop must return or raise") # pragma: no cover
class AsyncDynamicSession(AsyncSession, DynamicSessionMixin):
@@ -252,54 +259,59 @@ class AsyncDynamicSession(AsyncSession, DynamicSessionMixin):
else None
)
- page_info = await self._get_page(params.timeout, params.extra_headers, params.disable_resources)
- final_response = [None]
- handle_response = self._create_response_handler(page_info, final_response)
-
- if TYPE_CHECKING:
- from playwright.async_api import Page as async_Page
-
- if not isinstance(page_info.page, async_Page):
- raise TypeError
-
- try:
- # Navigate to URL and wait for a specified state
- page_info.page.on("response", handle_response)
- first_response = await page_info.page.goto(url, referer=referer)
- await self._wait_for_page_stability(page_info.page, params.load_dom, params.network_idle)
-
- if not first_response:
- raise RuntimeError(f"Failed to get response for {url}")
-
- if params.page_action:
- try:
- _ = await params.page_action(page_info.page)
- except Exception as e:
- log.error(f"Error executing page_action: {e}")
-
- if params.wait_selector:
- try:
- waiter: AsyncLocator = page_info.page.locator(params.wait_selector)
- await waiter.first.wait_for(state=params.wait_selector_state)
- # Wait again after waiting for the selector, helpful with protections like Cloudflare
- await self._wait_for_page_stability(page_info.page, params.load_dom, params.network_idle)
- except Exception as e:
- log.error(f"Error waiting for selector {params.wait_selector}: {e}")
-
- await page_info.page.wait_for_timeout(params.wait)
-
- # Create response object
- response = await ResponseFactory.from_async_playwright_response(
- page_info.page, first_response, final_response[0], params.selector_config
- )
-
- # Close the page to free up resources
- await page_info.page.close()
- self.page_pool.pages.remove(page_info)
- return response
-
- except Exception as e: # pragma: no cover
- page_info.mark_error()
- await page_info.page.close()
- self.page_pool.pages.remove(page_info)
- raise e
+ for attempt in range(self._config.retries):
+ page_info = await self._get_page(params.timeout, params.extra_headers, params.disable_resources)
+ final_response = [None]
+ handle_response = self._create_response_handler(page_info, final_response)
+
+ if TYPE_CHECKING:
+ from playwright.async_api import Page as async_Page
+
+ if not isinstance(page_info.page, async_Page):
+ raise TypeError
+
+ try:
+ page_info.page.on("response", handle_response)
+ first_response = await page_info.page.goto(url, referer=referer)
+ await self._wait_for_page_stability(page_info.page, params.load_dom, params.network_idle)
+
+ if not first_response:
+ raise RuntimeError(f"Failed to get response for {url}")
+
+ if params.page_action:
+ try:
+ _ = await params.page_action(page_info.page)
+ except Exception as e:
+ log.error(f"Error executing page_action: {e}")
+
+ if params.wait_selector:
+ try:
+ waiter: AsyncLocator = page_info.page.locator(params.wait_selector)
+ await waiter.first.wait_for(state=params.wait_selector_state)
+ await self._wait_for_page_stability(page_info.page, params.load_dom, params.network_idle)
+ except Exception as e:
+ log.error(f"Error waiting for selector {params.wait_selector}: {e}")
+
+ await page_info.page.wait_for_timeout(params.wait)
+
+ response = await ResponseFactory.from_async_playwright_response(
+ page_info.page, first_response, final_response[0], params.selector_config
+ )
+
+ await page_info.page.close()
+ self.page_pool.pages.remove(page_info)
+ return response
+
+ except Exception as e: # pragma: no cover
+ page_info.mark_error()
+ await page_info.page.close()
+ self.page_pool.pages.remove(page_info)
+
+ if attempt < self._config.retries - 1 and self._is_retriable(e):
+ log.warning(f"Attempt {attempt + 1} failed: {e}. Retrying in {self._config.retry_delay}s...")
+ await asyncio_sleep(self._config.retry_delay)
+ else:
+ raise
+
+ # For type checking purposes only
+ raise AssertionError("Unreachable: retry loop must return or raise") # pragma: no cover
diff --git a/scrapling/engines/_browsers/_stealth.py b/scrapling/engines/_browsers/_stealth.py
index ade3ab9..27c4d79 100644
--- a/scrapling/engines/_browsers/_stealth.py
+++ b/scrapling/engines/_browsers/_stealth.py
@@ -1,5 +1,7 @@
from random import randint
from re import compile as re_compile
+from time import sleep as time_sleep
+from asyncio import sleep as asyncio_sleep
from playwright.sync_api import (
Locator,
@@ -202,57 +204,66 @@ class StealthySession(SyncSession, StealthySessionMixin):
else None
)
- page_info = self._get_page(params.timeout, params.extra_headers, params.disable_resources)
- final_response = [None]
- handle_response = self._create_response_handler(page_info, final_response)
+ for attempt in range(self._config.retries):
+ page_info = self._get_page(params.timeout, params.extra_headers, params.disable_resources)
+ final_response = [None]
+ handle_response = self._create_response_handler(page_info, final_response)
- try: # pragma: no cover
- # Navigate to URL and wait for a specified state
- page_info.page.on("response", handle_response)
- first_response = page_info.page.goto(url, referer=referer)
- self._wait_for_page_stability(page_info.page, params.load_dom, params.network_idle)
-
- if not first_response:
- raise RuntimeError(f"Failed to get response for {url}")
-
- if params.solve_cloudflare:
- self._cloudflare_solver(page_info.page)
- # Make sure the page is fully loaded after the captcha
+ try: # pragma: no cover
+ # Navigate to URL and wait for a specified state
+ page_info.page.on("response", handle_response)
+ first_response = page_info.page.goto(url, referer=referer)
self._wait_for_page_stability(page_info.page, params.load_dom, params.network_idle)
- if params.page_action:
- try:
- _ = params.page_action(page_info.page)
- except Exception as e: # pragma: no cover
- log.error(f"Error executing page_action: {e}")
-
- if params.wait_selector:
- try:
- waiter: Locator = page_info.page.locator(params.wait_selector)
- waiter.first.wait_for(state=params.wait_selector_state)
- # Wait again after waiting for the selector, helpful with protections like Cloudflare
+ if not first_response:
+ raise RuntimeError(f"Failed to get response for {url}")
+
+ if params.solve_cloudflare:
+ self._cloudflare_solver(page_info.page)
+ # Make sure the page is fully loaded after the captcha
self._wait_for_page_stability(page_info.page, params.load_dom, params.network_idle)
- except Exception as e: # pragma: no cover
- log.error(f"Error waiting for selector {params.wait_selector}: {e}")
- page_info.page.wait_for_timeout(params.wait)
+ if params.page_action:
+ try:
+ _ = params.page_action(page_info.page)
+ except Exception as e: # pragma: no cover
+ log.error(f"Error executing page_action: {e}")
+
+ if params.wait_selector:
+ try:
+ waiter: Locator = page_info.page.locator(params.wait_selector)
+ waiter.first.wait_for(state=params.wait_selector_state)
+ # Wait again after waiting for the selector, helpful with protections like Cloudflare
+ self._wait_for_page_stability(page_info.page, params.load_dom, params.network_idle)
+ except Exception as e: # pragma: no cover
+ log.error(f"Error waiting for selector {params.wait_selector}: {e}")
+
+ page_info.page.wait_for_timeout(params.wait)
+
+ # Create response object
+ response = ResponseFactory.from_playwright_response(
+ page_info.page, first_response, final_response[0], params.selector_config
+ )
- # Create response object
- response = ResponseFactory.from_playwright_response(
- page_info.page, first_response, final_response[0], params.selector_config
- )
+ # Close the page to free up resources
+ page_info.page.close()
+ self.page_pool.pages.remove(page_info)
- # Close the page to free up resources
- page_info.page.close()
- self.page_pool.pages.remove(page_info)
+ return response
- return response
+ except Exception as e:
+ page_info.mark_error()
+ page_info.page.close()
+ self.page_pool.pages.remove(page_info)
- except Exception as e:
- page_info.mark_error()
- page_info.page.close()
- self.page_pool.pages.remove(page_info)
- raise e
+ if attempt < self._config.retries - 1 and self._is_retriable(e):
+ log.warning(f"Attempt {attempt + 1} failed: {e}. Retrying in {self._config.retry_delay}s...")
+ time_sleep(self._config.retry_delay)
+ else:
+ raise
+
+ # For type checking purposes only
+ raise AssertionError("Unreachable: retry loop must return or raise") # pragma: no cover
class AsyncStealthySession(AsyncSession, StealthySessionMixin):
@@ -422,53 +433,62 @@ class AsyncStealthySession(AsyncSession, StealthySessionMixin):
else None
)
- page_info = await self._get_page(params.timeout, params.extra_headers, params.disable_resources)
- final_response = [None]
- handle_response = self._create_response_handler(page_info, final_response)
-
- try:
- # Navigate to URL and wait for a specified state
- page_info.page.on("response", handle_response)
- first_response = await page_info.page.goto(url, referer=referer)
- await self._wait_for_page_stability(page_info.page, params.load_dom, params.network_idle)
-
- if not first_response:
- raise RuntimeError(f"Failed to get response for {url}")
+ for attempt in range(self._config.retries):
+ page_info = await self._get_page(params.timeout, params.extra_headers, params.disable_resources)
+ final_response = [None]
+ handle_response = self._create_response_handler(page_info, final_response)
- if params.solve_cloudflare:
- await self._cloudflare_solver(page_info.page)
- # Make sure the page is fully loaded after the captcha
+ try:
+ # Navigate to URL and wait for a specified state
+ page_info.page.on("response", handle_response)
+ first_response = await page_info.page.goto(url, referer=referer)
await self._wait_for_page_stability(page_info.page, params.load_dom, params.network_idle)
- if params.page_action:
- try:
- _ = await params.page_action(page_info.page)
- except Exception as e:
- log.error(f"Error executing page_action: {e}")
-
- if params.wait_selector:
- try:
- waiter: AsyncLocator = page_info.page.locator(params.wait_selector)
- await waiter.first.wait_for(state=params.wait_selector_state)
- # Wait again after waiting for the selector, helpful with protections like Cloudflare
+ if not first_response:
+ raise RuntimeError(f"Failed to get response for {url}")
+
+ if params.solve_cloudflare:
+ await self._cloudflare_solver(page_info.page)
+ # Make sure the page is fully loaded after the captcha
await self._wait_for_page_stability(page_info.page, params.load_dom, params.network_idle)
- except Exception as e:
- log.error(f"Error waiting for selector {params.wait_selector}: {e}")
-
- await page_info.page.wait_for_timeout(params.wait)
-
- # Create response object
- response = await ResponseFactory.from_async_playwright_response(
- page_info.page, first_response, final_response[0], params.selector_config
- )
-
- # Close the page to free up resources
- await page_info.page.close()
- self.page_pool.pages.remove(page_info)
- return response
-
- except Exception as e: # pragma: no cover
- page_info.mark_error()
- await page_info.page.close()
- self.page_pool.pages.remove(page_info)
- raise e
+
+ if params.page_action:
+ try:
+ _ = await params.page_action(page_info.page)
+ except Exception as e:
+ log.error(f"Error executing page_action: {e}")
+
+ if params.wait_selector:
+ try:
+ waiter: AsyncLocator = page_info.page.locator(params.wait_selector)
+ await waiter.first.wait_for(state=params.wait_selector_state)
+ # Wait again after waiting for the selector, helpful with protections like Cloudflare
+ await self._wait_for_page_stability(page_info.page, params.load_dom, params.network_idle)
+ except Exception as e:
+ log.error(f"Error waiting for selector {params.wait_selector}: {e}")
+
+ await page_info.page.wait_for_timeout(params.wait)
+
+ # Create response object
+ response = await ResponseFactory.from_async_playwright_response(
+ page_info.page, first_response, final_response[0], params.selector_config
+ )
+
+ # Close the page to free up resources
+ await page_info.page.close()
+ self.page_pool.pages.remove(page_info)
+ return response
+
+ except Exception as e: # pragma: no cover
+ page_info.mark_error()
+ await page_info.page.close()
+ self.page_pool.pages.remove(page_info)
+
+ if attempt < self._config.retries - 1 and self._is_retriable(e):
+ log.warning(f"Attempt {attempt + 1} failed: {e}. Retrying in {self._config.retry_delay}s...")
+ await asyncio_sleep(self._config.retry_delay)
+ else:
+ raise
+
+ # For type checking purposes only
+ raise AssertionError("Unreachable: retry loop must return or raise") # pragma: no cover
diff --git a/scrapling/engines/_browsers/_types.py b/scrapling/engines/_browsers/_types.py
index cb8d760..8391e9c 100644
--- a/scrapling/engines/_browsers/_types.py
+++ b/scrapling/engines/_browsers/_types.py
@@ -79,6 +79,8 @@ if TYPE_CHECKING: # pragma: no cover
cdp_url: Optional[str]
useragent: Optional[str]
extra_flags: Optional[List[str]]
+ retries: int
+ retry_delay: int | float
class PlaywrightFetchParams(TypedDict, total=False):
load_dom: bool
diff --git a/scrapling/engines/_browsers/_validators.py b/scrapling/engines/_browsers/_validators.py
index 82270ef..0b08f2f 100644
--- a/scrapling/engines/_browsers/_validators.py
+++ b/scrapling/engines/_browsers/_validators.py
@@ -50,6 +50,7 @@ def _is_invalid_cdp_url(cdp_url: str) -> bool | str:
# Type aliases for cleaner annotations
PagesCount = Annotated[int, Meta(ge=1, le=50)]
+RetriesCount = Annotated[int, Meta(ge=1, le=10)]
Seconds = Annotated[int, float, Meta(ge=0)]
@@ -80,6 +81,8 @@ class PlaywrightConfig(Struct, kw_only=True, frozen=False, weakref=True):
cdp_url: Optional[str] = None
useragent: Optional[str] = None
extra_flags: Optional[List[str]] = None
+ retries: RetriesCount = 3
+ retry_delay: Seconds = 1
def __post_init__(self): # pragma: no cover
"""Custom validation after msgspec validation"""