🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev
Python aiai-scrapingautomationcrawlercrawlingcrawling-pythondatadata-extractionmcpmcp-serverplaywrightpythonscrapingselectorsstealthweb-scraperweb-scrapingweb-scraping-pythonwebscrapingxpath
refactor/fix(browser fetchers): Solving a logic bug in page rotation
Karim shoairecaec6504959
diff --git a/scrapling/engines/_browsers/_base.py b/scrapling/engines/_browsers/_base.pynew file mode 100644index 0000000..55b3310--- /dev/null+++ b/scrapling/engines/_browsers/_base.py@@ -0,0 +1,403 @@+from time import time, sleep+from asyncio import sleep as asyncio_sleep, Lock++from camoufox import DefaultAddons+from playwright.sync_api import BrowserContext, Playwright+from playwright.async_api import (+ BrowserContext as AsyncBrowserContext,+ Playwright as AsyncPlaywright,+)+from camoufox.utils import (+ launch_options as generate_launch_options,+ installed_verstr as camoufox_version,+)++from scrapling.engines.toolbelt import (+ intercept_route,+ async_intercept_route,+ get_os_name,+)+from ._page import PageInfo, PagePool+from ._config_tools import _compiled_stealth_scripts+from ._validators import validate, PlaywrightConfig, CamoufoxConfig+from ._config_tools import _launch_kwargs, _context_kwargs+from scrapling.core._types import (+ Dict,+ Optional,+)++__ff_version_str__ = camoufox_version().split(".", 1)[0]+++class SyncSession:+ def __init__(self, max_pages: int = 1):+ self.max_pages = max_pages+ self.page_pool = PagePool(max_pages)+ self.__max_wait_for_page = 60+ self.playwright: Optional[Playwright] = None+ self.context: Optional[BrowserContext] = None+ self._closed = False++ def _get_page(self) -> PageInfo: # pragma: no cover+ """Get a new page to use"""++ # Close all finished pages to ensure clean state+ self.page_pool.close_all_finished_pages()++ # If we're at max capacity after cleanup, wait for busy pages to finish+ if self.page_pool.pages_count >= self.max_pages:+ start_time = time()+ while time() - start_time < self.__max_wait_for_page:+ # Wait for any pages to finish, then clean them up+ sleep(0.05)+ self.page_pool.close_all_finished_pages()+ if self.page_pool.pages_count < self.max_pages:+ break+ else:+ raise TimeoutError(+ f"No pages finished to clear place in the pool within the {self.__max_wait_for_page}s timeout period"+ )++ page = self.context.new_page()+ timeout = getattr(self, "timeout", 30000)+ page.set_default_navigation_timeout(timeout)+ page.set_default_timeout(timeout)+ if getattr(self, "extra_headers", False):+ page.set_extra_http_headers(getattr(self, "extra_headers"))++ if getattr(self, "disable_resources", False):+ page.route("**/*", intercept_route)++ if getattr(self, "stealth", False):+ for script in _compiled_stealth_scripts():+ page.add_init_script(script=script)++ return self.page_pool.add_page(page)++ def get_pool_stats(self) -> Dict[str, int]:+ """Get statistics about the current page pool"""+ return {+ "total_pages": self.page_pool.pages_count,+ "busy_pages": self.page_pool.busy_count,+ "max_pages": self.max_pages,+ }+++class AsyncSession(SyncSession):+ def __init__(self, max_pages: int = 1):+ super().__init__(max_pages)+ self.playwright: Optional[AsyncPlaywright] = None+ self.context: Optional[AsyncBrowserContext] = None+ self._lock = Lock()++ async def _get_page(self) -> PageInfo: # pragma: no cover+ """Get a new page to use"""+ async with self._lock:+ # Close all finished pages to ensure clean state+ await self.page_pool.aclose_all_finished_pages()++ # If we're at max capacity after cleanup, wait for busy pages to finish+ if self.page_pool.pages_count >= self.max_pages:+ start_time = time()+ while time() - start_time < self.__max_wait_for_page:+ # Wait for any pages to finish, then clean them up+ await asyncio_sleep(0.05)+ await self.page_pool.aclose_all_finished_pages()+ if self.page_pool.pages_count < self.max_pages:+ break+ else:+ raise TimeoutError(+ f"No pages finished to clear place in the pool within the {self.__max_wait_for_page}s timeout period"+ )++ page = await self.context.new_page()+ timeout = getattr(self, "timeout", 30000)+ page.set_default_navigation_timeout(timeout)+ page.set_default_timeout(timeout)+ if getattr(self, "extra_headers", False):+ await page.set_extra_http_headers(getattr(self, "extra_headers"))++ if getattr(self, "disable_resources", False):+ await page.route("**/*", async_intercept_route)++ if getattr(self, "stealth", False):+ for script in _compiled_stealth_scripts():+ await page.add_init_script(script=script)++ return self.page_pool.add_page(page)+++class DynamicSessionMixin:+ def __validate__(+ self,+ __max_pages,+ headless,+ google_search,+ hide_canvas,+ disable_webgl,+ real_chrome,+ stealth,+ wait,+ page_action,+ proxy,+ locale,+ extra_headers,+ useragent,+ cdp_url,+ timeout,+ disable_resources,+ wait_selector,+ init_script,+ cookies,+ network_idle,+ wait_selector_state,+ selector_config,+ ):+ params = {+ "max_pages": __max_pages,+ "headless": headless,+ "google_search": google_search,+ "hide_canvas": hide_canvas,+ "disable_webgl": disable_webgl,+ "real_chrome": real_chrome,+ "stealth": stealth,+ "wait": wait,+ "page_action": page_action,+ "proxy": proxy,+ "locale": locale,+ "extra_headers": extra_headers,+ "useragent": useragent,+ "timeout": timeout,+ "selector_config": selector_config,+ "disable_resources": disable_resources,+ "wait_selector": wait_selector,+ "init_script": init_script,+ "cookies": cookies,+ "network_idle": network_idle,+ "wait_selector_state": wait_selector_state,+ "cdp_url": cdp_url,+ }+ config = validate(params, PlaywrightConfig)++ self.max_pages = config.max_pages+ self.headless = config.headless+ self.hide_canvas = config.hide_canvas+ self.disable_webgl = config.disable_webgl+ self.real_chrome = config.real_chrome+ self.stealth = config.stealth+ self.google_search = config.google_search+ self.wait = config.wait+ self.proxy = config.proxy+ self.locale = config.locale+ self.extra_headers = config.extra_headers+ self.useragent = config.useragent+ self.timeout = config.timeout+ self.cookies = config.cookies+ self.disable_resources = config.disable_resources+ self.cdp_url = config.cdp_url+ self.network_idle = config.network_idle+ self.wait_selector = config.wait_selector+ self.init_script = config.init_script+ self.wait_selector_state = config.wait_selector_state+ self.selector_config = config.selector_config+ self.page_action = config.page_action+ self._headers_keys = (+ set(map(str.lower, self.extra_headers.keys()))+ if self.extra_headers+ else set()+ )+ self.__initiate_browser_options__()++ def __initiate_browser_options__(self):+ if not self.cdp_url:+ # `launch_options` is used with persistent context+ self.launch_options = dict(+ _launch_kwargs(+ self.headless,+ self.proxy,+ self.locale,+ tuple(self.extra_headers.items())+ if self.extra_headers+ else tuple(),+ self.useragent,+ self.real_chrome,+ self.stealth,+ self.hide_canvas,+ self.disable_webgl,+ )+ )+ self.launch_options["extra_http_headers"] = dict(+ self.launch_options["extra_http_headers"]+ )+ self.launch_options["proxy"] = dict(self.launch_options["proxy"]) or None+ self.context_options = dict()+ else:+ # while `context_options` is left to be used when cdp mode is enabled+ self.launch_options = dict()+ self.context_options = dict(+ _context_kwargs(+ self.proxy,+ self.locale,+ tuple(self.extra_headers.items())+ if self.extra_headers+ else tuple(),+ self.useragent,+ self.stealth,+ )+ )+ self.context_options["extra_http_headers"] = dict(+ self.context_options["extra_http_headers"]+ )+ self.context_options["proxy"] = dict(self.context_options["proxy"]) or None+++class StealthySessionMixin:+ def __validate__(+ self,+ max_pages,+ headless,+ block_images,+ disable_resources,+ block_webrtc,+ allow_webgl,+ network_idle,+ humanize,+ solve_cloudflare,+ wait,+ timeout,+ page_action,+ wait_selector,+ init_script,+ addons,+ wait_selector_state,+ cookies,+ google_search,+ extra_headers,+ proxy,+ os_randomize,+ disable_ads,+ geoip,+ selector_config,+ additional_args,+ ):+ params = {+ "max_pages": max_pages,+ "headless": headless,+ "block_images": block_images,+ "disable_resources": disable_resources,+ "block_webrtc": block_webrtc,+ "allow_webgl": allow_webgl,+ "network_idle": network_idle,+ "humanize": humanize,+ "solve_cloudflare": solve_cloudflare,+ "wait": wait,+ "timeout": timeout,+ "page_action": page_action,+ "wait_selector": wait_selector,+ "init_script": init_script,+ "addons": addons,+ "wait_selector_state": wait_selector_state,+ "cookies": cookies,+ "google_search": google_search,+ "extra_headers": extra_headers,+ "proxy": proxy,+ "os_randomize": os_randomize,+ "disable_ads": disable_ads,+ "geoip": geoip,+ "selector_config": selector_config,+ "additional_args": additional_args,+ }+ config = validate(params, CamoufoxConfig)++ self.max_pages = config.max_pages+ self.headless = config.headless+ self.block_images = config.block_images+ self.disable_resources = config.disable_resources+ self.block_webrtc = config.block_webrtc+ self.allow_webgl = config.allow_webgl+ self.network_idle = config.network_idle+ self.humanize = config.humanize+ self.solve_cloudflare = config.solve_cloudflare+ self.wait = config.wait+ self.timeout = config.timeout+ self.page_action = config.page_action+ self.wait_selector = config.wait_selector+ self.init_script = config.init_script+ self.addons = config.addons+ self.wait_selector_state = config.wait_selector_state+ self.cookies = config.cookies+ self.google_search = config.google_search+ self.extra_headers = config.extra_headers+ self.proxy = config.proxy+ self.os_randomize = config.os_randomize+ self.disable_ads = config.disable_ads+ self.geoip = config.geoip+ self.selector_config = config.selector_config+ self.additional_args = config.additional_args+ self.selector_config = config.selector_config+ self.page_action = config.page_action+ self._headers_keys = (+ set(map(str.lower, self.extra_headers.keys()))+ if self.extra_headers+ else set()+ )+ self.__initiate_browser_options__()++ def __initiate_browser_options__(self):+ """Initiate browser options."""+ self.launch_options = generate_launch_options(+ **{+ "geoip": self.geoip,+ "proxy": dict(self.proxy) if self.proxy else self.proxy,+ "addons": self.addons,+ "exclude_addons": [] if self.disable_ads else [DefaultAddons.UBO],+ "headless": self.headless,+ "humanize": True if self.solve_cloudflare else self.humanize,+ "i_know_what_im_doing": True, # To turn warnings off with the user configurations+ "allow_webgl": self.allow_webgl,+ "block_webrtc": self.block_webrtc,+ "block_images": self.block_images, # Careful! it makes some websites don't finish loading at all like stackoverflow even in headful mode.+ "os": None if self.os_randomize else get_os_name(),+ "user_data_dir": "",+ "ff_version": __ff_version_str__,+ "firefox_user_prefs": {+ # This is what enabling `enable_cache` does internally, so we do it from here instead+ "browser.sessionhistory.max_entries": 10,+ "browser.sessionhistory.max_total_viewers": -1,+ "browser.cache.memory.enable": True,+ "browser.cache.disk_cache_ssl": True,+ "browser.cache.disk.smart_size.enabled": True,+ },+ **self.additional_args,+ }+ )++ @staticmethod+ def _detect_cloudflare(page_content: str) -> str | None:+ """+ Detect the type of Cloudflare challenge present in the provided page content.++ This function analyzes the given page content to identify whether a specific+ type of Cloudflare challenge is present. It checks for three predefined+ challenge types: non-interactive, managed, and interactive. If a challenge+ type is detected, it returns the corresponding type as a string. If no+ challenge type is detected, it returns None.++ Args:+ page_content (str): The content of the page to analyze for Cloudflare+ challenge types.++ Returns:+ str: A string representing the detected Cloudflare challenge type, if+ found. Returns None if no challenge matches.+ """+ challenge_types = (+ "non-interactive",+ "managed",+ "interactive",+ )+ for ctype in challenge_types:+ if f"cType: '{ctype}'" in page_content:+ return ctype++ return Nonediff --git a/scrapling/engines/_browsers/_camoufox.py b/scrapling/engines/_browsers/_camoufox.pyindex 5afccb1..cd814c1 100644--- a/scrapling/engines/_browsers/_camoufox.py+++ b/scrapling/engines/_browsers/_camoufox.py@@ -1,17 +1,8 @@-from time import time, sleepfrom re import compile as re_compile-from asyncio import sleep as asyncio_sleep, Lock-from camoufox import DefaultAddons-from camoufox.utils import (- launch_options as generate_launch_options,- installed_verstr as camoufox_version,-)from playwright.sync_api import (Response as SyncPlaywrightResponse,sync_playwright,- BrowserContext,- Playwright,Locator,Page,)@@ -24,9 +15,8 @@ from playwright.async_api import (Page as async_Page,)+from ._base import SyncSession, AsyncSession, StealthySessionMixinfrom scrapling.core.utils import log-from ._page import PageInfo, PagePool-from ._validators import validate, CamoufoxConfigfrom scrapling.core._types import (Dict,List,@@ -37,17 +27,13 @@ from scrapling.core._types import (from scrapling.engines.toolbelt import (Response,ResponseFactory,- async_intercept_route,generate_convincing_referer,- get_os_name,- intercept_route,)__CF_PATTERN__ = re_compile("challenges.cloudflare.com/cdn-cgi/challenge-platform/.*")-__ff_version_str__ = camoufox_version().split(".", 1)[0]-class StealthySession:+class StealthySession(StealthySessionMixin, SyncSession):"""A Stealthy session manager with page pooling."""__slots__ = (@@ -87,7 +73,7 @@ class StealthySession:def __init__(self,- max_pages: int = 1,+ __max_pages: int = 1,headless: bool = True, # noqa: F821block_images: bool = False,disable_resources: bool = False,@@ -141,107 +127,38 @@ class StealthySession::param google_search: Enabled by default, Scrapling will set the referer header to be as if this request came from a Google search of this website's domain name.:param extra_headers: A dictionary of extra headers to add to the request. _The referer set by the `google_search` argument takes priority over the referer set here if used together._:param proxy: The proxy to be used with requests, it can be a string or a dictionary with the keys 'server', 'username', and 'password' only.- :param max_pages: The maximum number of tabs to be opened at the same time. It will be used in rotation through a PagePool.:param selector_config: The arguments that will be passed in the end while creating the final Selector's class.:param additional_args: Additional arguments to be passed to Camoufox as additional settings, and it takes higher priority than Scrapling's settings."""- params = {- "max_pages": max_pages,- "headless": headless,- "block_images": block_images,- "disable_resources": disable_resources,- "block_webrtc": block_webrtc,- "allow_webgl": allow_webgl,- "network_idle": network_idle,- "humanize": humanize,- "solve_cloudflare": solve_cloudflare,- "wait": wait,- "timeout": timeout,- "page_action": page_action,- "wait_selector": wait_selector,- "init_script": init_script,- "addons": addons,- "wait_selector_state": wait_selector_state,- "cookies": cookies,- "google_search": google_search,- "extra_headers": extra_headers,- "proxy": proxy,- "os_randomize": os_randomize,- "disable_ads": disable_ads,- "geoip": geoip,- "selector_config": selector_config,- "additional_args": additional_args,- }- config = validate(params, CamoufoxConfig)-- self.max_pages = config.max_pages- self.headless = config.headless- self.block_images = config.block_images- self.disable_resources = config.disable_resources- self.block_webrtc = config.block_webrtc- self.allow_webgl = config.allow_webgl- self.network_idle = config.network_idle- self.humanize = config.humanize- self.solve_cloudflare = config.solve_cloudflare- self.wait = config.wait- self.timeout = config.timeout- self.page_action = config.page_action- self.wait_selector = config.wait_selector- self.init_script = config.init_script- self.addons = config.addons- self.wait_selector_state = config.wait_selector_state- self.cookies = config.cookies- self.google_search = config.google_search- self.extra_headers = config.extra_headers- self.proxy = config.proxy- self.os_randomize = config.os_randomize- self.disable_ads = config.disable_ads- self.geoip = config.geoip- self.selector_config = config.selector_config- self.additional_args = config.additional_args-- self.playwright: Optional[Playwright] = None- self.context: Optional[BrowserContext] = None- self.page_pool = PagePool(self.max_pages)- self._closed = False- self.selector_config = config.selector_config- self.page_action = config.page_action- self._headers_keys = (- set(map(str.lower, self.extra_headers.keys()))- if self.extra_headers- else set()- )- self.__initiate_browser_options__()-- def __initiate_browser_options__(self):- """Initiate browser options."""- self.launch_options = generate_launch_options(- **{- "geoip": self.geoip,- "proxy": dict(self.proxy) if self.proxy else self.proxy,- "addons": self.addons,- "exclude_addons": [] if self.disable_ads else [DefaultAddons.UBO],- "headless": self.headless,- "humanize": True if self.solve_cloudflare else self.humanize,- "i_know_what_im_doing": True, # To turn warnings off with the user configurations- "allow_webgl": self.allow_webgl,- "block_webrtc": self.block_webrtc,- "block_images": self.block_images, # Careful! it makes some websites don't finish loading at all like stackoverflow even in headful mode.- "os": None if self.os_randomize else get_os_name(),- "user_data_dir": "",- "ff_version": __ff_version_str__,- "firefox_user_prefs": {- # This is what enabling `enable_cache` does internally, so we do it from here instead- "browser.sessionhistory.max_entries": 10,- "browser.sessionhistory.max_total_viewers": -1,- "browser.cache.memory.enable": True,- "browser.cache.disk_cache_ssl": True,- "browser.cache.disk.smart_size.enabled": True,- },- **self.additional_args,- }+ self.__validate__(+ __max_pages,+ headless,+ block_images,+ disable_resources,+ block_webrtc,+ allow_webgl,+ network_idle,+ humanize,+ solve_cloudflare,+ wait,+ timeout,+ page_action,+ wait_selector,+ init_script,+ addons,+ wait_selector_state,+ cookies,+ google_search,+ extra_headers,+ proxy,+ os_randomize,+ disable_ads,+ geoip,+ selector_config,+ additional_args,)+ super().__init__(max_pages=self.max_pages)def __create__(self):"""Create a browser for this instance and context."""@@ -284,68 +201,6 @@ class StealthySession:self._closed = True- def _get_or_create_page(self) -> PageInfo: # pragma: no cover- """Get an available page or create a new one"""- # Try to get a ready page first- page_info = self.page_pool.get_ready_page()- if page_info:- return page_info-- # Create a new page if under limit- if self.page_pool.pages_count < self.max_pages:- page = self.context.new_page()- page.set_default_navigation_timeout(self.timeout)- page.set_default_timeout(self.timeout)- if self.extra_headers:- page.set_extra_http_headers(self.extra_headers)-- if self.disable_resources:- page.route("**/*", intercept_route)-- return self.page_pool.add_page(page)-- # Wait for a page to become available- max_wait = 30- start_time = time()-- while time() - start_time < max_wait:- page_info = self.page_pool.get_ready_page()- if page_info:- return page_info- sleep(0.05)-- raise TimeoutError("No pages available within timeout period")-- @staticmethod- def _detect_cloudflare(page_content):- """- Detect the type of Cloudflare challenge present in the provided page content.-- This function analyzes the given page content to identify whether a specific- type of Cloudflare challenge is present. It checks for three predefined- challenge types: non-interactive, managed, and interactive. If a challenge- type is detected, it returns the corresponding type as a string. If no- challenge type is detected, it returns None.-- Args:- page_content (str): The content of the page to analyze for Cloudflare- challenge types.-- Returns:- str: A string representing the detected Cloudflare challenge type, if- found. Returns None if no challenge matches.- """- challenge_types = (- "non-interactive",- "managed",- "interactive",- )- for ctype in challenge_types:- if f"cType: '{ctype}'" in page_content:- return ctype-- return None-def _solve_cloudflare(self, page: Page) -> None: # pragma: no cover"""Solve the cloudflare challenge displayed on the playwright page passed@@ -416,7 +271,7 @@ class StealthySession:):final_response = finished_response- page_info = self._get_or_create_page()+ page_info = self._get_page()page_info.mark_busy(url=url)try: # pragma: no cover@@ -462,8 +317,8 @@ class StealthySession:page_info.page, first_response, final_response, self.selector_config)- # Mark the page as ready for next use- page_info.mark_ready()+ # Mark the page as finished for next use+ page_info.mark_finished()return response@@ -471,17 +326,8 @@ class StealthySession:page_info.mark_error()raise e- def get_pool_stats(self) -> Dict[str, int]:- """Get statistics about the current page pool"""- return {- "total_pages": self.page_pool.pages_count,- "ready_pages": self.page_pool.ready_count,- "busy_pages": self.page_pool.busy_count,- "max_pages": self.max_pages,- }--class AsyncStealthySession(StealthySession):+class AsyncStealthySession(StealthySessionMixin, AsyncSession):"""A Stealthy session manager with page pooling."""def __init__(@@ -544,7 +390,7 @@ class AsyncStealthySession(StealthySession)::param selector_config: The arguments that will be passed in the end while creating the final Selector's class.:param additional_args: Additional arguments to be passed to Camoufox as additional settings, and it takes higher priority than Scrapling's settings."""- super().__init__(+ self.__validate__(max_pages,headless,block_images,@@ -571,11 +417,7 @@ class AsyncStealthySession(StealthySession):selector_config,additional_args,)- self.playwright: Optional[AsyncPlaywright] = None- self.context: Optional[AsyncBrowserContext] = None- self._lock = Lock()- self.__enter__ = None- self.__exit__ = None+ super().__init__(max_pages=self.max_pages)async def __create__(self):"""Create a browser for this instance and context."""@@ -618,39 +460,6 @@ class AsyncStealthySession(StealthySession):self._closed = True- async def _get_or_create_page(self) -> PageInfo:- """Get an available page or create a new one"""- async with self._lock:- # Try to get a ready page first- page_info = self.page_pool.get_ready_page()- if page_info:- return page_info-- # Create a new page if under limit- if self.page_pool.pages_count < self.max_pages:- page = await self.context.new_page()- page.set_default_navigation_timeout(self.timeout)- page.set_default_timeout(self.timeout)- if self.extra_headers:- await page.set_extra_http_headers(self.extra_headers)-- if self.disable_resources:- await page.route("**/*", async_intercept_route)-- return self.page_pool.add_page(page)-- # Wait for a page to become available- max_wait = 30- start_time = time()-- while time() - start_time < max_wait: # pragma: no cover- page_info = self.page_pool.get_ready_page()- if page_info:- return page_info- await asyncio_sleep(0.05)-- raise TimeoutError("No pages available within timeout period")-async def _solve_cloudflare(self, page: async_Page):"""Solve the cloudflare challenge displayed on the playwright page passed. The async version@@ -723,7 +532,7 @@ class AsyncStealthySession(StealthySession):):final_response = finished_response- page_info = await self._get_or_create_page()+ page_info = await self._get_page()page_info.mark_busy(url=url)try:@@ -771,8 +580,8 @@ class AsyncStealthySession(StealthySession):page_info.page, first_response, final_response, self.selector_config)- # Mark the page as ready for next use- page_info.mark_ready()+ # Mark the page as finished for next use+ page_info.mark_finished()return responsediff --git a/scrapling/engines/_browsers/_controllers.py b/scrapling/engines/_browsers/_controllers.pyindex 85db198..ef3b906 100644--- a/scrapling/engines/_browsers/_controllers.py+++ b/scrapling/engines/_browsers/_controllers.py@@ -1,10 +1,6 @@-from time import time, sleep-from asyncio import sleep as asyncio_sleep, Lock-from playwright.sync_api import (Response as SyncPlaywrightResponse,sync_playwright,- BrowserContext,Playwright,Locator,)@@ -21,9 +17,7 @@ from rebrowser_playwright.async_api import ()from scrapling.core.utils import log-from ._page import PageInfo, PagePool-from ._validators import validate, PlaywrightConfig-from ._config_tools import _compiled_stealth_scripts, _launch_kwargs, _context_kwargs+from ._base import SyncSession, AsyncSession, DynamicSessionMixinfrom scrapling.core._types import (Dict,List,@@ -35,12 +29,10 @@ from scrapling.engines.toolbelt import (Response,ResponseFactory,generate_convincing_referer,- intercept_route,- async_intercept_route,)-class DynamicSession:+class DynamicSession(DynamicSessionMixin, SyncSession):"""A Browser session manager with page pooling."""__slots__ = (@@ -127,108 +119,31 @@ class DynamicSession::param proxy: The proxy to be used with requests, it can be a string or a dictionary with the keys 'server', 'username', and 'password' only.:param selector_config: The arguments that will be passed in the end while creating the final Selector's class."""-- params = {- "max_pages": __max_pages,- "headless": headless,- "google_search": google_search,- "hide_canvas": hide_canvas,- "disable_webgl": disable_webgl,- "real_chrome": real_chrome,- "stealth": stealth,- "wait": wait,- "page_action": page_action,- "proxy": proxy,- "locale": locale,- "extra_headers": extra_headers,- "useragent": useragent,- "timeout": timeout,- "selector_config": selector_config,- "disable_resources": disable_resources,- "wait_selector": wait_selector,- "init_script": init_script,- "cookies": cookies,- "network_idle": network_idle,- "wait_selector_state": wait_selector_state,- "cdp_url": cdp_url,- }- config = validate(params, PlaywrightConfig)-- self.max_pages = config.max_pages- self.headless = config.headless- self.hide_canvas = config.hide_canvas- self.disable_webgl = config.disable_webgl- self.real_chrome = config.real_chrome- self.stealth = config.stealth- self.google_search = config.google_search- self.wait = config.wait- self.proxy = config.proxy- self.locale = config.locale- self.extra_headers = config.extra_headers- self.useragent = config.useragent- self.timeout = config.timeout- self.cookies = config.cookies- self.disable_resources = config.disable_resources- self.cdp_url = config.cdp_url- self.network_idle = config.network_idle- self.wait_selector = config.wait_selector- self.init_script = config.init_script- self.wait_selector_state = config.wait_selector_state-- self.playwright: Optional[Playwright] = None- self.context: Optional[BrowserContext] = None- self.page_pool = PagePool(self.max_pages)- self._closed = False- self.selector_config = config.selector_config- self.page_action = config.page_action- self._headers_keys = (- set(map(str.lower, self.extra_headers.keys()))- if self.extra_headers- else set()+ self.__validate__(+ __max_pages,+ headless,+ google_search,+ hide_canvas,+ disable_webgl,+ real_chrome,+ stealth,+ wait,+ page_action,+ proxy,+ locale,+ extra_headers,+ useragent,+ cdp_url,+ timeout,+ disable_resources,+ wait_selector,+ init_script,+ cookies,+ network_idle,+ wait_selector_state,+ selector_config,)- self.__initiate_browser_options__()-- def __initiate_browser_options__(self):- if not self.cdp_url:- # `launch_options` is used with persistent context- self.launch_options = dict(- _launch_kwargs(- self.headless,- self.proxy,- self.locale,- tuple(self.extra_headers.items())- if self.extra_headers- else tuple(),- self.useragent,- self.real_chrome,- self.stealth,- self.hide_canvas,- self.disable_webgl,- )- )- self.launch_options["extra_http_headers"] = dict(- self.launch_options["extra_http_headers"]- )- self.launch_options["proxy"] = dict(self.launch_options["proxy"]) or None- self.context_options = dict()- else:- # while `context_options` is left to be used when cdp mode is enabled- self.launch_options = dict()- self.context_options = dict(- _context_kwargs(- self.proxy,- self.locale,- tuple(self.extra_headers.items())- if self.extra_headers- else tuple(),- self.useragent,- self.stealth,- )- )- self.context_options["extra_http_headers"] = dict(- self.context_options["extra_http_headers"]- )- self.context_options["proxy"] = dict(self.context_options["proxy"]) or None+ super().__init__(max_pages=self.max_pages)def __create__(self):"""Create a browser for this instance and context."""@@ -237,7 +152,7 @@ class DynamicSession:# Because rebrowser_playwright doesn't play well with real browserssync_context = sync_playwright- self.playwright = sync_context().start()+ self.playwright: Playwright = sync_context().start()if self.cdp_url: # pragma: no coverself.context = self.playwright.chromium.connect_over_cdp(@@ -280,43 +195,10 @@ class DynamicSession:self._closed = True- def _get_or_create_page(self) -> PageInfo: # pragma: no cover- """Get an available page or create a new one"""- # Try to get a ready page first- page_info = self.page_pool.get_ready_page()- if page_info:- return page_info-- # Create a new page if under limit- if self.page_pool.pages_count < self.max_pages:- page = self.context.new_page()- page.set_default_navigation_timeout(self.timeout)- page.set_default_timeout(self.timeout)- if self.extra_headers:- page.set_extra_http_headers(self.extra_headers)-- if self.disable_resources:- page.route("**/*", intercept_route)-- if self.stealth:- for script in _compiled_stealth_scripts():- page.add_init_script(script=script)-- return self.page_pool.add_page(page)-- # Wait for a page to become available- max_wait = 30- start_time = time()-- while time() - start_time < max_wait:- page_info = self.page_pool.get_ready_page()- if page_info:- return page_info- sleep(0.05)-- raise TimeoutError("No pages available within timeout period")-- def fetch(self, url: str) -> Response:+ def fetch(+ self,+ url: str,+ ) -> Response:"""Opens up the browser and do your request based on your chosen options.:param url: The Target url.@@ -340,7 +222,7 @@ class DynamicSession:):final_response = finished_response- page_info = self._get_or_create_page()+ page_info = self._get_page()page_info.mark_busy(url=url)try: # pragma: no cover@@ -380,8 +262,8 @@ class DynamicSession:page_info.page, first_response, final_response, self.selector_config)- # Mark the page as ready for next use- page_info.mark_ready()+ # Mark the page as finished for next use+ page_info.mark_finished()return response@@ -389,17 +271,8 @@ class DynamicSession:page_info.mark_error()raise e- def get_pool_stats(self) -> Dict[str, int]:- """Get statistics about the current page pool"""- return {- "total_pages": self.page_pool.pages_count,- "ready_pages": self.page_pool.ready_count,- "busy_pages": self.page_pool.busy_count,- "max_pages": self.max_pages,- }--class AsyncDynamicSession(DynamicSession):+class AsyncDynamicSession(DynamicSessionMixin, AsyncSession):"""An async Browser session manager with page pooling, it's using a persistent browser Context by default with a temporary user profile directory."""def __init__(@@ -455,7 +328,7 @@ class AsyncDynamicSession(DynamicSession)::param selector_config: The arguments that will be passed in the end while creating the final Selector's class."""- super().__init__(+ self.__validate__(max_pages,headless,google_search,@@ -479,12 +352,7 @@ class AsyncDynamicSession(DynamicSession):wait_selector_state,selector_config,)-- self.playwright: Optional[AsyncPlaywright] = None- self.context: Optional[AsyncBrowserContext] = None- self._lock = Lock()- self.__enter__ = None- self.__exit__ = None+ super().__init__(max_pages=self.max_pages)async def __create__(self):"""Create a browser for this instance and context."""@@ -541,43 +409,6 @@ class AsyncDynamicSession(DynamicSession):self._closed = True- async def _get_or_create_page(self) -> PageInfo:- """Get an available page or create a new one"""- async with self._lock:- # Try to get a ready page first- page_info = self.page_pool.get_ready_page()- if page_info:- return page_info-- # Create a new page if under limit- if self.page_pool.pages_count < self.max_pages:- page = await self.context.new_page()- page.set_default_navigation_timeout(self.timeout)- page.set_default_timeout(self.timeout)- if self.extra_headers:- await page.set_extra_http_headers(self.extra_headers)-- if self.disable_resources:- await page.route("**/*", async_intercept_route)-- if self.stealth:- for script in _compiled_stealth_scripts():- await page.add_init_script(script=script)-- return self.page_pool.add_page(page)-- # Wait for a page to become available- max_wait = 30 # seconds- start_time = time()-- while time() - start_time < max_wait: # pragma: no cover- page_info = self.page_pool.get_ready_page()- if page_info:- return page_info- await asyncio_sleep(0.05)-- raise TimeoutError("No pages available within timeout period")-async def fetch(self, url: str) -> Response:"""Opens up the browser and do your request based on your chosen options.@@ -602,7 +433,7 @@ class AsyncDynamicSession(DynamicSession):):final_response = finished_response- page_info = await self._get_or_create_page()+ page_info = await self._get_page()page_info.mark_busy(url=url)try:@@ -642,8 +473,8 @@ class AsyncDynamicSession(DynamicSession):page_info.page, first_response, final_response, self.selector_config)- # Mark the page as ready for next use- page_info.mark_ready()+ # Mark the page as finished for next use+ page_info.mark_finished()return responsediff --git a/scrapling/engines/_browsers/_page.py b/scrapling/engines/_browsers/_page.pyindex ec418d0..61dd51b 100644--- a/scrapling/engines/_browsers/_page.py+++ b/scrapling/engines/_browsers/_page.py@@ -6,7 +6,9 @@ from playwright.async_api import Page as AsyncPagefrom scrapling.core._types import Optional, List, Literal-PageState = Literal["ready", "busy", "error"] # States that a page can be in+PageState = Literal[+ "finished", "ready", "busy", "error"+] # States that a page can be in@dataclass@@ -23,9 +25,9 @@ class PageInfo:self.state = "busy"self.url = url- def mark_ready(self):- """Mark the page as ready for new requests"""- self.state = "ready"+ def mark_finished(self):+ """Mark the page as finished for new requests"""+ self.state = "finished"self.url = ""def mark_error(self):@@ -62,24 +64,16 @@ class PagePool:self.pages.append(page_info)return page_info- def get_ready_page(self) -> Optional[PageInfo]:- """Get a page that's ready for use"""- with self._lock:- for page_info in self.pages:- if page_info.state == "ready":- return page_info- return None-@propertydef pages_count(self) -> int:"""Get the total number of pages"""return len(self.pages)@property- def ready_count(self) -> int:- """Get the number of ready pages"""+ def finished_count(self) -> int:+ """Get the number of finished pages"""with self._lock:- return sum(1 for p in self.pages if p.state == "ready")+ return sum(1 for p in self.pages if p.state == "finished")@propertydef busy_count(self) -> int:@@ -91,3 +85,33 @@ class PagePool:"""Remove pages in error state"""with self._lock:self.pages = [p for p in self.pages if p.state != "error"]++ def close_all_finished_pages(self):+ """Close all pages in finished state and remove them from the pool"""+ with self._lock:+ pages_to_remove = []+ for page_info in self.pages:+ if page_info.state == "finished":+ try:+ page_info.page.close()+ except Exception:+ pass+ pages_to_remove.append(page_info)++ for page_info in pages_to_remove:+ self.pages.remove(page_info)++ async def aclose_all_finished_pages(self):+ """Async version: Close all pages in finished state and remove them from the pool"""+ with self._lock:+ pages_to_remove = []+ for page_info in self.pages:+ if page_info.state == "finished":+ try:+ await page_info.page.close()+ except Exception:+ pass+ pages_to_remove.append(page_info)++ for page_info in pages_to_remove:+ self.pages.remove(page_info)diff --git a/scrapling/fetchers.py b/scrapling/fetchers.pyindex c1ce8f7..3e39ba2 100644--- a/scrapling/fetchers.py+++ b/scrapling/fetchers.py@@ -118,7 +118,6 @@ class StealthyFetcher(BaseFetcher):with StealthySession(wait=wait,- max_pages=1,proxy=proxy,geoip=geoip,addons=addons,