🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev
Python aiai-scrapingautomationcrawlercrawlingcrawling-pythondatadata-extractionmcpmcp-serverplaywrightpythonscrapingselectorsstealthweb-scraperweb-scrapingweb-scraping-pythonwebscrapingxpath
feat(browsers): Add option to retry tabs that gives errors
Karim shoaira17bb1d97607
diff --git a/scrapling/engines/_browsers/_base.py b/scrapling/engines/_browsers/_base.pyindex cbbdc24..810c51c 100644--- a/scrapling/engines/_browsers/_base.py+++ b/scrapling/engines/_browsers/_base.py@@ -310,6 +310,14 @@ class BaseSessionMixin:if config.additional_args:self._context_options.update(config.additional_args)+ @staticmethod+ def _is_retriable(error: Exception) -> bool:+ """Check if an error is retriable (transient network/timeout issues)."""+ if isinstance(error, TimeoutError):+ return True+ error_msg = str(error).lower()+ return "net::" in error_msg or "failed to get response" in error_msg+class DynamicSessionMixin(BaseSessionMixin):def __validate__(self, **params):diff --git a/scrapling/engines/_browsers/_controllers.py b/scrapling/engines/_browsers/_controllers.pyindex 3d39e9d..bd5d455 100644--- a/scrapling/engines/_browsers/_controllers.py+++ b/scrapling/engines/_browsers/_controllers.py@@ -1,3 +1,6 @@+from time import sleep as time_sleep+from asyncio import sleep as asyncio_sleep+from playwright.sync_api import (Locator,Playwright,@@ -115,52 +118,56 @@ class DynamicSession(SyncSession, DynamicSessionMixin):else None)- page_info = self._get_page(params.timeout, params.extra_headers, params.disable_resources)- final_response = [None]- handle_response = self._create_response_handler(page_info, final_response)-- try: # pragma: no cover- # Navigate to URL and wait for a specified state- page_info.page.on("response", handle_response)- first_response = page_info.page.goto(url, referer=referer)- self._wait_for_page_stability(page_info.page, params.load_dom, params.network_idle)-- if not first_response:- raise RuntimeError(f"Failed to get response for {url}")-- if params.page_action:- try:- _ = params.page_action(page_info.page)- except Exception as e: # pragma: no cover- log.error(f"Error executing page_action: {e}")-- if params.wait_selector:- try:- waiter: Locator = page_info.page.locator(params.wait_selector)- waiter.first.wait_for(state=params.wait_selector_state)- # Wait again after waiting for the selector, helpful with protections like Cloudflare- self._wait_for_page_stability(page_info.page, params.load_dom, params.network_idle)- except Exception as e: # pragma: no cover- log.error(f"Error waiting for selector {params.wait_selector}: {e}")-- page_info.page.wait_for_timeout(params.wait)+ for attempt in range(self._config.retries):+ page_info = self._get_page(params.timeout, params.extra_headers, params.disable_resources)+ final_response = [None]+ handle_response = self._create_response_handler(page_info, final_response)++ try: # pragma: no cover+ page_info.page.on("response", handle_response)+ first_response = page_info.page.goto(url, referer=referer)+ self._wait_for_page_stability(page_info.page, params.load_dom, params.network_idle)++ if not first_response:+ raise RuntimeError(f"Failed to get response for {url}")++ if params.page_action:+ try:+ _ = params.page_action(page_info.page)+ except Exception as e: # pragma: no cover+ log.error(f"Error executing page_action: {e}")++ if params.wait_selector:+ try:+ waiter: Locator = page_info.page.locator(params.wait_selector)+ waiter.first.wait_for(state=params.wait_selector_state)+ self._wait_for_page_stability(page_info.page, params.load_dom, params.network_idle)+ except Exception as e: # pragma: no cover+ log.error(f"Error waiting for selector {params.wait_selector}: {e}")++ page_info.page.wait_for_timeout(params.wait)++ response = ResponseFactory.from_playwright_response(+ page_info.page, first_response, final_response[0], params.selector_config+ )- # Create response object- response = ResponseFactory.from_playwright_response(- page_info.page, first_response, final_response[0], params.selector_config- )+ page_info.page.close()+ self.page_pool.pages.remove(page_info)+ return response- # Close the page to free up resources- page_info.page.close()- self.page_pool.pages.remove(page_info)+ except Exception as e:+ page_info.mark_error()+ page_info.page.close()+ self.page_pool.pages.remove(page_info)- return response+ if attempt < self._config.retries - 1 and self._is_retriable(e):+ log.warning(f"Attempt {attempt + 1} failed: {e}. Retrying in {self._config.retry_delay}s...")+ time_sleep(self._config.retry_delay)+ else:+ raise- except Exception as e:- page_info.mark_error()- page_info.page.close()- self.page_pool.pages.remove(page_info)- raise e+ # For type checking purposes only+ raise AssertionError("Unreachable: retry loop must return or raise") # pragma: no coverclass AsyncDynamicSession(AsyncSession, DynamicSessionMixin):@@ -252,54 +259,59 @@ class AsyncDynamicSession(AsyncSession, DynamicSessionMixin):else None)- page_info = await self._get_page(params.timeout, params.extra_headers, params.disable_resources)- final_response = [None]- handle_response = self._create_response_handler(page_info, final_response)-- if TYPE_CHECKING:- from playwright.async_api import Page as async_Page-- if not isinstance(page_info.page, async_Page):- raise TypeError-- try:- # Navigate to URL and wait for a specified state- page_info.page.on("response", handle_response)- first_response = await page_info.page.goto(url, referer=referer)- await self._wait_for_page_stability(page_info.page, params.load_dom, params.network_idle)-- if not first_response:- raise RuntimeError(f"Failed to get response for {url}")-- if params.page_action:- try:- _ = await params.page_action(page_info.page)- except Exception as e:- log.error(f"Error executing page_action: {e}")-- if params.wait_selector:- try:- waiter: AsyncLocator = page_info.page.locator(params.wait_selector)- await waiter.first.wait_for(state=params.wait_selector_state)- # Wait again after waiting for the selector, helpful with protections like Cloudflare- await self._wait_for_page_stability(page_info.page, params.load_dom, params.network_idle)- except Exception as e:- log.error(f"Error waiting for selector {params.wait_selector}: {e}")-- await page_info.page.wait_for_timeout(params.wait)-- # Create response object- response = await ResponseFactory.from_async_playwright_response(- page_info.page, first_response, final_response[0], params.selector_config- )-- # Close the page to free up resources- await page_info.page.close()- self.page_pool.pages.remove(page_info)- return response-- except Exception as e: # pragma: no cover- page_info.mark_error()- await page_info.page.close()- self.page_pool.pages.remove(page_info)- raise e+ for attempt in range(self._config.retries):+ page_info = await self._get_page(params.timeout, params.extra_headers, params.disable_resources)+ final_response = [None]+ handle_response = self._create_response_handler(page_info, final_response)++ if TYPE_CHECKING:+ from playwright.async_api import Page as async_Page++ if not isinstance(page_info.page, async_Page):+ raise TypeError++ try:+ page_info.page.on("response", handle_response)+ first_response = await page_info.page.goto(url, referer=referer)+ await self._wait_for_page_stability(page_info.page, params.load_dom, params.network_idle)++ if not first_response:+ raise RuntimeError(f"Failed to get response for {url}")++ if params.page_action:+ try:+ _ = await params.page_action(page_info.page)+ except Exception as e:+ log.error(f"Error executing page_action: {e}")++ if params.wait_selector:+ try:+ waiter: AsyncLocator = page_info.page.locator(params.wait_selector)+ await waiter.first.wait_for(state=params.wait_selector_state)+ await self._wait_for_page_stability(page_info.page, params.load_dom, params.network_idle)+ except Exception as e:+ log.error(f"Error waiting for selector {params.wait_selector}: {e}")++ await page_info.page.wait_for_timeout(params.wait)++ response = await ResponseFactory.from_async_playwright_response(+ page_info.page, first_response, final_response[0], params.selector_config+ )++ await page_info.page.close()+ self.page_pool.pages.remove(page_info)+ return response++ except Exception as e: # pragma: no cover+ page_info.mark_error()+ await page_info.page.close()+ self.page_pool.pages.remove(page_info)++ if attempt < self._config.retries - 1 and self._is_retriable(e):+ log.warning(f"Attempt {attempt + 1} failed: {e}. Retrying in {self._config.retry_delay}s...")+ await asyncio_sleep(self._config.retry_delay)+ else:+ raise++ # For type checking purposes only+ raise AssertionError("Unreachable: retry loop must return or raise") # pragma: no coverdiff --git a/scrapling/engines/_browsers/_stealth.py b/scrapling/engines/_browsers/_stealth.pyindex ade3ab9..27c4d79 100644--- a/scrapling/engines/_browsers/_stealth.py+++ b/scrapling/engines/_browsers/_stealth.py@@ -1,5 +1,7 @@from random import randintfrom re import compile as re_compile+from time import sleep as time_sleep+from asyncio import sleep as asyncio_sleepfrom playwright.sync_api import (Locator,@@ -202,57 +204,66 @@ class StealthySession(SyncSession, StealthySessionMixin):else None)- page_info = self._get_page(params.timeout, params.extra_headers, params.disable_resources)- final_response = [None]- handle_response = self._create_response_handler(page_info, final_response)+ for attempt in range(self._config.retries):+ page_info = self._get_page(params.timeout, params.extra_headers, params.disable_resources)+ final_response = [None]+ handle_response = self._create_response_handler(page_info, final_response)- try: # pragma: no cover- # Navigate to URL and wait for a specified state- page_info.page.on("response", handle_response)- first_response = page_info.page.goto(url, referer=referer)- self._wait_for_page_stability(page_info.page, params.load_dom, params.network_idle)-- if not first_response:- raise RuntimeError(f"Failed to get response for {url}")-- if params.solve_cloudflare:- self._cloudflare_solver(page_info.page)- # Make sure the page is fully loaded after the captcha+ try: # pragma: no cover+ # Navigate to URL and wait for a specified state+ page_info.page.on("response", handle_response)+ first_response = page_info.page.goto(url, referer=referer)self._wait_for_page_stability(page_info.page, params.load_dom, params.network_idle)- if params.page_action:- try:- _ = params.page_action(page_info.page)- except Exception as e: # pragma: no cover- log.error(f"Error executing page_action: {e}")-- if params.wait_selector:- try:- waiter: Locator = page_info.page.locator(params.wait_selector)- waiter.first.wait_for(state=params.wait_selector_state)- # Wait again after waiting for the selector, helpful with protections like Cloudflare+ if not first_response:+ raise RuntimeError(f"Failed to get response for {url}")++ if params.solve_cloudflare:+ self._cloudflare_solver(page_info.page)+ # Make sure the page is fully loaded after the captchaself._wait_for_page_stability(page_info.page, params.load_dom, params.network_idle)- except Exception as e: # pragma: no cover- log.error(f"Error waiting for selector {params.wait_selector}: {e}")- page_info.page.wait_for_timeout(params.wait)+ if params.page_action:+ try:+ _ = params.page_action(page_info.page)+ except Exception as e: # pragma: no cover+ log.error(f"Error executing page_action: {e}")++ if params.wait_selector:+ try:+ waiter: Locator = page_info.page.locator(params.wait_selector)+ waiter.first.wait_for(state=params.wait_selector_state)+ # Wait again after waiting for the selector, helpful with protections like Cloudflare+ self._wait_for_page_stability(page_info.page, params.load_dom, params.network_idle)+ except Exception as e: # pragma: no cover+ log.error(f"Error waiting for selector {params.wait_selector}: {e}")++ page_info.page.wait_for_timeout(params.wait)++ # Create response object+ response = ResponseFactory.from_playwright_response(+ page_info.page, first_response, final_response[0], params.selector_config+ )- # Create response object- response = ResponseFactory.from_playwright_response(- page_info.page, first_response, final_response[0], params.selector_config- )+ # Close the page to free up resources+ page_info.page.close()+ self.page_pool.pages.remove(page_info)- # Close the page to free up resources- page_info.page.close()- self.page_pool.pages.remove(page_info)+ return response- return response+ except Exception as e:+ page_info.mark_error()+ page_info.page.close()+ self.page_pool.pages.remove(page_info)- except Exception as e:- page_info.mark_error()- page_info.page.close()- self.page_pool.pages.remove(page_info)- raise e+ if attempt < self._config.retries - 1 and self._is_retriable(e):+ log.warning(f"Attempt {attempt + 1} failed: {e}. Retrying in {self._config.retry_delay}s...")+ time_sleep(self._config.retry_delay)+ else:+ raise++ # For type checking purposes only+ raise AssertionError("Unreachable: retry loop must return or raise") # pragma: no coverclass AsyncStealthySession(AsyncSession, StealthySessionMixin):@@ -422,53 +433,62 @@ class AsyncStealthySession(AsyncSession, StealthySessionMixin):else None)- page_info = await self._get_page(params.timeout, params.extra_headers, params.disable_resources)- final_response = [None]- handle_response = self._create_response_handler(page_info, final_response)-- try:- # Navigate to URL and wait for a specified state- page_info.page.on("response", handle_response)- first_response = await page_info.page.goto(url, referer=referer)- await self._wait_for_page_stability(page_info.page, params.load_dom, params.network_idle)-- if not first_response:- raise RuntimeError(f"Failed to get response for {url}")+ for attempt in range(self._config.retries):+ page_info = await self._get_page(params.timeout, params.extra_headers, params.disable_resources)+ final_response = [None]+ handle_response = self._create_response_handler(page_info, final_response)- if params.solve_cloudflare:- await self._cloudflare_solver(page_info.page)- # Make sure the page is fully loaded after the captcha+ try:+ # Navigate to URL and wait for a specified state+ page_info.page.on("response", handle_response)+ first_response = await page_info.page.goto(url, referer=referer)await self._wait_for_page_stability(page_info.page, params.load_dom, params.network_idle)- if params.page_action:- try:- _ = await params.page_action(page_info.page)- except Exception as e:- log.error(f"Error executing page_action: {e}")-- if params.wait_selector:- try:- waiter: AsyncLocator = page_info.page.locator(params.wait_selector)- await waiter.first.wait_for(state=params.wait_selector_state)- # Wait again after waiting for the selector, helpful with protections like Cloudflare+ if not first_response:+ raise RuntimeError(f"Failed to get response for {url}")++ if params.solve_cloudflare:+ await self._cloudflare_solver(page_info.page)+ # Make sure the page is fully loaded after the captchaawait self._wait_for_page_stability(page_info.page, params.load_dom, params.network_idle)- except Exception as e:- log.error(f"Error waiting for selector {params.wait_selector}: {e}")-- await page_info.page.wait_for_timeout(params.wait)-- # Create response object- response = await ResponseFactory.from_async_playwright_response(- page_info.page, first_response, final_response[0], params.selector_config- )-- # Close the page to free up resources- await page_info.page.close()- self.page_pool.pages.remove(page_info)- return response-- except Exception as e: # pragma: no cover- page_info.mark_error()- await page_info.page.close()- self.page_pool.pages.remove(page_info)- raise e++ if params.page_action:+ try:+ _ = await params.page_action(page_info.page)+ except Exception as e:+ log.error(f"Error executing page_action: {e}")++ if params.wait_selector:+ try:+ waiter: AsyncLocator = page_info.page.locator(params.wait_selector)+ await waiter.first.wait_for(state=params.wait_selector_state)+ # Wait again after waiting for the selector, helpful with protections like Cloudflare+ await self._wait_for_page_stability(page_info.page, params.load_dom, params.network_idle)+ except Exception as e:+ log.error(f"Error waiting for selector {params.wait_selector}: {e}")++ await page_info.page.wait_for_timeout(params.wait)++ # Create response object+ response = await ResponseFactory.from_async_playwright_response(+ page_info.page, first_response, final_response[0], params.selector_config+ )++ # Close the page to free up resources+ await page_info.page.close()+ self.page_pool.pages.remove(page_info)+ return response++ except Exception as e: # pragma: no cover+ page_info.mark_error()+ await page_info.page.close()+ self.page_pool.pages.remove(page_info)++ if attempt < self._config.retries - 1 and self._is_retriable(e):+ log.warning(f"Attempt {attempt + 1} failed: {e}. Retrying in {self._config.retry_delay}s...")+ await asyncio_sleep(self._config.retry_delay)+ else:+ raise++ # For type checking purposes only+ raise AssertionError("Unreachable: retry loop must return or raise") # pragma: no coverdiff --git a/scrapling/engines/_browsers/_types.py b/scrapling/engines/_browsers/_types.pyindex cb8d760..8391e9c 100644--- a/scrapling/engines/_browsers/_types.py+++ b/scrapling/engines/_browsers/_types.py@@ -79,6 +79,8 @@ if TYPE_CHECKING: # pragma: no covercdp_url: Optional[str]useragent: Optional[str]extra_flags: Optional[List[str]]+ retries: int+ retry_delay: int | floatclass PlaywrightFetchParams(TypedDict, total=False):load_dom: booldiff --git a/scrapling/engines/_browsers/_validators.py b/scrapling/engines/_browsers/_validators.pyindex 82270ef..0b08f2f 100644--- a/scrapling/engines/_browsers/_validators.py+++ b/scrapling/engines/_browsers/_validators.py@@ -50,6 +50,7 @@ def _is_invalid_cdp_url(cdp_url: str) -> bool | str:# Type aliases for cleaner annotationsPagesCount = Annotated[int, Meta(ge=1, le=50)]+RetriesCount = Annotated[int, Meta(ge=1, le=10)]Seconds = Annotated[int, float, Meta(ge=0)]@@ -80,6 +81,8 @@ class PlaywrightConfig(Struct, kw_only=True, frozen=False, weakref=True):cdp_url: Optional[str] = Noneuseragent: Optional[str] = Noneextra_flags: Optional[List[str]] = None+ retries: RetriesCount = 3+ retry_delay: Seconds = 1def __post_init__(self): # pragma: no cover"""Custom validation after msgspec validation"""