🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev
Python aiai-scrapingautomationcrawlercrawlingcrawling-pythondatadata-extractionmcpmcp-serverplaywrightpythonscrapingselectorsstealthweb-scraperweb-scrapingweb-scraping-pythonwebscrapingxpath
refactor(browser fetchers): Make all the type hints dynamic + Faster validation
Karim shoairf0b01fc25333
diff --git a/scrapling/core/_types.py b/scrapling/core/_types.pyindex bd83f49..ac19a91 100644--- a/scrapling/core/_types.py+++ b/scrapling/core/_types.py@@ -12,6 +12,7 @@ from typing import (Callable,Dict,Generator,+ Generic,Iterable,List,Set,diff --git a/scrapling/engines/_browsers/_base.py b/scrapling/engines/_browsers/_base.pyindex f670e4c..f329fa4 100644--- a/scrapling/engines/_browsers/_base.py+++ b/scrapling/engines/_browsers/_base.py@@ -2,15 +2,15 @@ from time import timefrom asyncio import sleep as asyncio_sleep, Lockfrom camoufox import DefaultAddons+from playwright.sync_api._generated import Pagefrom playwright.sync_api import (- Page,Frame,BrowserContext,Playwright,Response as SyncPlaywrightResponse,)+from playwright.async_api._generated import Page as AsyncPagefrom playwright.async_api import (- Page as AsyncPage,Frame as AsyncFrame,Playwright as AsyncPlaywright,Response as AsyncPlaywrightResponse,@@ -70,7 +70,7 @@ class SyncSession:timeout: int | float,extra_headers: Optional[Dict[str, str]],disable_resources: bool,- ) -> PageInfo: # pragma: no cover+ ) -> PageInfo[Page]: # pragma: no cover"""Get a new page to use"""# No need to check if a page is available or not in sync code because the code blocked before reaching here till the page closed, ofc.@@ -116,7 +116,7 @@ class SyncSession:self._wait_for_networkidle(page)@staticmethod- def _create_response_handler(page_info: PageInfo, response_container: List) -> Callable:+ def _create_response_handler(page_info: PageInfo[Page], response_container: List) -> Callable:"""Create a response handler that captures the final navigation response.:param page_info: The PageInfo object containing the page@@ -175,7 +175,7 @@ class AsyncSession:timeout: int | float,extra_headers: Optional[Dict[str, str]],disable_resources: bool,- ) -> PageInfo: # pragma: no cover+ ) -> PageInfo[AsyncPage]: # pragma: no cover"""Get a new page to use"""if TYPE_CHECKING:assert self.context is not None, "Browser context not initialized"@@ -232,7 +232,7 @@ class AsyncSession:await self._wait_for_networkidle(page)@staticmethod- def _create_response_handler(page_info: PageInfo, response_container: List) -> Callable:+ def _create_response_handler(page_info: PageInfo[AsyncPage], response_container: List) -> Callable:"""Create an async response handler that captures the final navigation response.:param page_info: The PageInfo object containing the page@@ -253,130 +253,135 @@ class AsyncSession:class DynamicSessionMixin:def __validate__(self, **params):+ if "__max_pages" in params:+ params["max_pages"] = params.pop("__max_pages")+config = validate(params, model=PlaywrightConfig)- self.max_pages = config.max_pages- self.headless = config.headless- self.hide_canvas = config.hide_canvas- self.disable_webgl = config.disable_webgl- self.real_chrome = config.real_chrome- self.stealth = config.stealth- self.google_search = config.google_search- self.wait = config.wait- self.proxy = config.proxy- self.locale = config.locale- self.extra_headers = config.extra_headers- self.useragent = config.useragent- self.timeout = config.timeout- self.cookies = config.cookies- self.disable_resources = config.disable_resources- self.cdp_url = config.cdp_url- self.network_idle = config.network_idle- self.load_dom = config.load_dom- self.wait_selector = config.wait_selector- self.init_script = config.init_script- self.wait_selector_state = config.wait_selector_state- self.extra_flags = config.extra_flags- self.selector_config = config.selector_config- self.additional_args = config.additional_args- self.page_action = config.page_action- self.user_data_dir = config.user_data_dir- self._headers_keys = {header.lower() for header in self.extra_headers.keys()} if self.extra_headers else set()+ self._max_pages = config.max_pages+ self._headless = config.headless+ self._hide_canvas = config.hide_canvas+ self._disable_webgl = config.disable_webgl+ self._real_chrome = config.real_chrome+ self._stealth = config.stealth+ self._google_search = config.google_search+ self._wait = config.wait+ self._proxy = config.proxy+ self._locale = config.locale+ self._extra_headers = config.extra_headers+ self._useragent = config.useragent+ self._timeout = config.timeout+ self._cookies = config.cookies+ self._disable_resources = config.disable_resources+ self._cdp_url = config.cdp_url+ self._network_idle = config.network_idle+ self._load_dom = config.load_dom+ self._wait_selector = config.wait_selector+ self._init_script = config.init_script+ self._wait_selector_state = config.wait_selector_state+ self._extra_flags = config.extra_flags+ self._selector_config = config.selector_config+ self._additional_args = config.additional_args+ self._page_action = config.page_action+ self._user_data_dir = config.user_data_dir+ self._headers_keys = {header.lower() for header in self._extra_headers.keys()} if self._extra_headers else set()self.__initiate_browser_options__()def __initiate_browser_options__(self):if TYPE_CHECKING:- assert isinstance(self.proxy, tuple)+ assert isinstance(self._proxy, tuple)- if not self.cdp_url:+ if not self._cdp_url:# `launch_options` is used with persistent contextself.launch_options = dict(_launch_kwargs(- self.headless,- self.proxy,- self.locale,- tuple(self.extra_headers.items()) if self.extra_headers else tuple(),- self.useragent,- self.real_chrome,- self.stealth,- self.hide_canvas,- self.disable_webgl,- tuple(self.extra_flags) if self.extra_flags else tuple(),+ self._headless,+ self._proxy,+ self._locale,+ tuple(self._extra_headers.items()) if self._extra_headers else tuple(),+ self._useragent,+ self._real_chrome,+ self._stealth,+ self._hide_canvas,+ self._disable_webgl,+ tuple(self._extra_flags) if self._extra_flags else tuple(),))self.launch_options["extra_http_headers"] = dict(self.launch_options["extra_http_headers"])self.launch_options["proxy"] = dict(self.launch_options["proxy"]) or None- self.launch_options["user_data_dir"] = self.user_data_dir- self.launch_options.update(cast(Dict, self.additional_args))+ self.launch_options["user_data_dir"] = self._user_data_dir+ self.launch_options.update(cast(Dict, self._additional_args))self.context_options = dict()else:# while `context_options` is left to be used when cdp mode is enabledself.launch_options = dict()self.context_options = dict(_context_kwargs(- self.proxy,- self.locale,- tuple(self.extra_headers.items()) if self.extra_headers else tuple(),- self.useragent,- self.stealth,+ self._proxy,+ self._locale,+ tuple(self._extra_headers.items()) if self._extra_headers else tuple(),+ self._useragent,+ self._stealth,))self.context_options["extra_http_headers"] = dict(self.context_options["extra_http_headers"])self.context_options["proxy"] = dict(self.context_options["proxy"]) or None- self.context_options.update(cast(Dict, self.additional_args))+ self.context_options.update(cast(Dict, self._additional_args))class StealthySessionMixin:def __validate__(self, **params):+ if "__max_pages" in params:+ params["max_pages"] = params.pop("__max_pages")+config: CamoufoxConfig = validate(params, model=CamoufoxConfig)- self.max_pages = config.max_pages- self.headless = config.headless- self.block_images = config.block_images- self.disable_resources = config.disable_resources- self.block_webrtc = config.block_webrtc- self.allow_webgl = config.allow_webgl- self.network_idle = config.network_idle- self.load_dom = config.load_dom- self.humanize = config.humanize- self.solve_cloudflare = config.solve_cloudflare- self.wait = config.wait- self.timeout = config.timeout- self.page_action = config.page_action- self.wait_selector = config.wait_selector- self.init_script = config.init_script- self.addons = config.addons- self.wait_selector_state = config.wait_selector_state- self.cookies = config.cookies- self.google_search = config.google_search- self.extra_headers = config.extra_headers- self.proxy = config.proxy- self.os_randomize = config.os_randomize- self.disable_ads = config.disable_ads- self.geoip = config.geoip- self.selector_config = config.selector_config- self.additional_args = config.additional_args- self.page_action = config.page_action- self.user_data_dir = config.user_data_dir- self._headers_keys = {header.lower() for header in self.extra_headers.keys()} if self.extra_headers else set()+ self._max_pages = config.max_pages+ self._headless = config.headless+ self._block_images = config.block_images+ self._disable_resources = config.disable_resources+ self._block_webrtc = config.block_webrtc+ self._allow_webgl = config.allow_webgl+ self._network_idle = config.network_idle+ self._load_dom = config.load_dom+ self._humanize = config.humanize+ self._solve_cloudflare = config.solve_cloudflare+ self._wait = config.wait+ self._timeout = config.timeout+ self._page_action = config.page_action+ self._wait_selector = config.wait_selector+ self._init_script = config.init_script+ self._addons = config.addons+ self._wait_selector_state = config.wait_selector_state+ self._cookies = config.cookies+ self._google_search = config.google_search+ self._extra_headers = config.extra_headers+ self._proxy = config.proxy+ self._os_randomize = config.os_randomize+ self._disable_ads = config.disable_ads+ self._geoip = config.geoip+ self._selector_config = config.selector_config+ self._additional_args = config.additional_args+ self._user_data_dir = config.user_data_dir+ self._headers_keys = {header.lower() for header in self._extra_headers.keys()} if self._extra_headers else set()self.__initiate_browser_options__()def __initiate_browser_options__(self):"""Initiate browser options."""self.launch_options: Dict[str, Any] = generate_launch_options(**{- "geoip": self.geoip,- "proxy": dict(self.proxy) if self.proxy and isinstance(self.proxy, tuple) else self.proxy,- "addons": self.addons,- "exclude_addons": [] if self.disable_ads else [DefaultAddons.UBO],- "headless": self.headless,- "humanize": True if self.solve_cloudflare else self.humanize,+ "geoip": self._geoip,+ "proxy": dict(self._proxy) if self._proxy and isinstance(self._proxy, tuple) else self._proxy,+ "addons": self._addons,+ "exclude_addons": [] if self._disable_ads else [DefaultAddons.UBO],+ "headless": self._headless,+ "humanize": True if self._solve_cloudflare else self._humanize,"i_know_what_im_doing": True, # To turn warnings off with the user configurations- "allow_webgl": self.allow_webgl,- "block_webrtc": self.block_webrtc,- "block_images": self.block_images, # Careful! it makes some websites don't finish loading at all like stackoverflow even in headful mode.- "os": None if self.os_randomize else get_os_name(),- "user_data_dir": self.user_data_dir,+ "allow_webgl": self._allow_webgl,+ "block_webrtc": self._block_webrtc,+ "block_images": self._block_images, # Careful! it makes some websites don't finish loading at all like stackoverflow even in headful mode.+ "os": None if self._os_randomize else get_os_name(),+ "user_data_dir": self._user_data_dir,"ff_version": __ff_version_str__,"firefox_user_prefs": {# This is what enabling `enable_cache` does internally, so we do it from here instead@@ -386,7 +391,7 @@ class StealthySessionMixin:"browser.cache.disk_cache_ssl": True,"browser.cache.disk.smart_size.enabled": True,},- **cast(Dict, self.additional_args),+ **cast(Dict, self._additional_args),})diff --git a/scrapling/engines/_browsers/_camoufox.py b/scrapling/engines/_browsers/_camoufox.pyindex 8d70dbd..436a006 100644--- a/scrapling/engines/_browsers/_camoufox.py+++ b/scrapling/engines/_browsers/_camoufox.py@@ -14,58 +14,47 @@ from playwright.async_api import (BrowserContext as AsyncBrowserContext,)-from ._validators import validate_fetch as _validate, CamoufoxConfig-from ._base import SyncSession, AsyncSession, StealthySessionMixinfrom scrapling.core.utils import log-from scrapling.core._types import (- Any,- Dict,- List,- Optional,- Callable,- TYPE_CHECKING,- SelectorWaitStates,-)-from scrapling.engines.toolbelt.convertor import (- Response,- ResponseFactory,-)+from ._types import CamoufoxSession, CamoufoxFetchParams+from scrapling.core._types import Any, Unpack, TYPE_CHECKING+from ._base import SyncSession, AsyncSession, StealthySessionMixin+from ._validators import validate_fetch as _validate, CamoufoxConfig+from scrapling.engines.toolbelt.convertor import Response, ResponseFactoryfrom scrapling.engines.toolbelt.fingerprints import generate_convincing_referer__CF_PATTERN__ = re_compile("challenges.cloudflare.com/cdn-cgi/challenge-platform/.*")-_UNSET: Any = object()class StealthySession(StealthySessionMixin, SyncSession):"""A Stealthy session manager with page pooling."""__slots__ = (- "max_pages",- "headless",- "block_images",- "disable_resources",- "block_webrtc",- "allow_webgl",- "network_idle",- "load_dom",- "humanize",- "solve_cloudflare",- "wait",- "timeout",- "page_action",- "wait_selector",- "init_script",- "addons",- "wait_selector_state",- "cookies",- "google_search",- "extra_headers",- "proxy",- "os_randomize",- "disable_ads",- "geoip",- "selector_config",- "additional_args",+ "_max_pages",+ "_headless",+ "_block_images",+ "_disable_resources",+ "_block_webrtc",+ "_allow_webgl",+ "_network_idle",+ "_load_dom",+ "_humanize",+ "_solve_cloudflare",+ "_wait",+ "_timeout",+ "_page_action",+ "_wait_selector",+ "_init_script",+ "_addons",+ "_wait_selector_state",+ "_cookies",+ "_google_search",+ "_extra_headers",+ "_proxy",+ "_os_randomize",+ "_disable_ads",+ "_geoip",+ "_selector_config",+ "_additional_args","playwright","browser","context",@@ -73,38 +62,10 @@ class StealthySession(StealthySessionMixin, SyncSession):"_closed","launch_options","_headers_keys",+ "_user_data_dir",)- def __init__(- self,- __max_pages: int = 1,- headless: bool = True, # noqa: F821- block_images: bool = False,- disable_resources: bool = False,- block_webrtc: bool = False,- allow_webgl: bool = True,- network_idle: bool = False,- load_dom: bool = True,- humanize: bool | float = True,- solve_cloudflare: bool = False,- wait: int | float = 0,- timeout: int | float = 30000,- page_action: Optional[Callable] = None,- wait_selector: Optional[str] = None,- init_script: Optional[str] = None,- addons: Optional[List[str]] = None,- wait_selector_state: SelectorWaitStates = "attached",- cookies: Optional[List[Dict]] = None,- google_search: bool = True,- extra_headers: Optional[Dict[str, str]] = None,- proxy: Optional[str | Dict[str, str]] = None,- os_randomize: bool = False,- disable_ads: bool = False,- geoip: bool = False,- user_data_dir: str = "",- selector_config: Optional[Dict] = None,- additional_args: Optional[Dict] = None,- ):+ def __init__(self, **kwargs: Unpack[CamoufoxSession]):"""A Browser session manager with page pooling:param headless: Run the browser in headless/hidden (default), or headful/visible mode.@@ -138,50 +99,21 @@ class StealthySession(StealthySessionMixin, SyncSession)::param selector_config: The arguments that will be passed in the end while creating the final Selector's class.:param additional_args: Additional arguments to be passed to Camoufox as additional settings, and it takes higher priority than Scrapling's settings."""-- self.__validate__(- wait=wait,- proxy=proxy,- geoip=geoip,- addons=addons,- timeout=timeout,- cookies=cookies,- headless=headless,- humanize=humanize,- load_dom=load_dom,- max_pages=__max_pages,- disable_ads=disable_ads,- allow_webgl=allow_webgl,- page_action=page_action,- init_script=init_script,- network_idle=network_idle,- block_images=block_images,- block_webrtc=block_webrtc,- os_randomize=os_randomize,- user_data_dir=user_data_dir,- wait_selector=wait_selector,- google_search=google_search,- extra_headers=extra_headers,- additional_args=additional_args,- selector_config=selector_config,- solve_cloudflare=solve_cloudflare,- disable_resources=disable_resources,- wait_selector_state=wait_selector_state,- )- super().__init__(max_pages=self.max_pages)+ self.__validate__(**kwargs)+ super().__init__(max_pages=self._max_pages)def __create__(self):"""Create a browser for this instance and context."""self.playwright = sync_playwright().start()self.context = self.playwright.firefox.launch_persistent_context(**self.launch_options)- if self.init_script: # pragma: no cover- self.context.add_init_script(path=self.init_script)+ if self._init_script: # pragma: no cover+ self.context.add_init_script(path=self._init_script)- if self.cookies: # pragma: no cover- self.context.add_cookies(self.cookies)+ if self._cookies: # pragma: no cover+ self.context.add_cookies(self._cookies)- def _solve_cloudflare(self, page: Page) -> None: # pragma: no cover+ def _cloudflare_solver(self, page: Page) -> None: # pragma: no cover"""Solve the cloudflare challenge displayed on the playwright page passed:param page: The targeted page@@ -247,59 +179,28 @@ class StealthySession(StealthySessionMixin, SyncSession):log.info("Cloudflare captcha is solved")return- def fetch(- self,- url: str,- google_search: bool = _UNSET,- timeout: int | float = _UNSET,- wait: int | float = _UNSET,- page_action: Optional[Callable] = _UNSET,- extra_headers: Optional[Dict[str, str]] = _UNSET,- disable_resources: bool = _UNSET,- wait_selector: Optional[str] = _UNSET,- wait_selector_state: SelectorWaitStates = _UNSET,- network_idle: bool = _UNSET,- load_dom: bool = _UNSET,- solve_cloudflare: bool = _UNSET,- selector_config: Optional[Dict] = _UNSET,- ) -> Response:+ def fetch(self, url: str, **kwargs: Unpack[CamoufoxFetchParams]) -> Response:"""Opens up the browser and do your request based on your chosen options.:param url: The Target url.- :param google_search: Enabled by default, Scrapling will set the referer header to be as if this request came from a Google search of this website's domain name.- :param timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30,000- :param wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the ` Response ` object.- :param page_action: Added for automation. A function that takes the `page` object and does the automation you need.- :param extra_headers: A dictionary of extra headers to add to the request. _The referer set by the `google_search` argument takes priority over the referer set here if used together._- :param disable_resources: Drop requests of unnecessary resources for a speed boost. It depends, but it made requests ~25% faster in my tests for some websites.- Requests dropped are of type `font`, `image`, `media`, `beacon`, `object`, `imageset`, `texttrack`, `websocket`, `csp_report`, and `stylesheet`.- This can help save your proxy usage but be careful with this option as it makes some websites never finish loading.- :param wait_selector: Wait for a specific CSS selector to be in a specific state.- :param wait_selector_state: The state to wait for the selector given with `wait_selector`. The default state is `attached`.- :param network_idle: Wait for the page until there are no network connections for at least 500 ms.- :param load_dom: Enabled by default, wait for all JavaScript on page(s) to fully load and execute.- :param solve_cloudflare: Solves all types of the Cloudflare's Turnstile/Interstitial challenges before returning the response to you.- :param selector_config: The arguments that will be passed in the end while creating the final Selector's class.+ :param kwargs: Additional keyword arguments including:+ - google_search: Enabled by default, Scrapling will set the referer header to be as if this request came from a Google search of this website's domain name.+ - timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30,000+ - wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the ` Response ` object.+ - page_action: Added for automation. A function that takes the `page` object and does the automation you need.+ - extra_headers: A dictionary of extra headers to add to the request. _The referer set by the `google_search` argument takes priority over the referer set here if used together._+ - disable_resources: Drop requests of unnecessary resources for a speed boost. It depends, but it made requests ~25% faster in my tests for some websites.+ Requests dropped are of type `font`, `image`, `media`, `beacon`, `object`, `imageset`, `texttrack`, `websocket`, `csp_report`, and `stylesheet`.+ This can help save your proxy usage but be careful with this option as it makes some websites never finish loading.+ - wait_selector: Wait for a specific CSS selector to be in a specific state.+ - wait_selector_state: The state to wait for the selector given with `wait_selector`. The default state is `attached`.+ - network_idle: Wait for the page until there are no network connections for at least 500 ms.+ - load_dom: Enabled by default, wait for all JavaScript on page(s) to fully load and execute.+ - solve_cloudflare: Solves all types of the Cloudflare's Turnstile/Interstitial challenges before returning the response to you.+ - selector_config: The arguments that will be passed in the end while creating the final Selector's class.:return: A `Response` object."""- params = _validate(- [- ("google_search", google_search, self.google_search),- ("timeout", timeout, self.timeout),- ("wait", wait, self.wait),- ("page_action", page_action, self.page_action),- ("extra_headers", extra_headers, self.extra_headers),- ("disable_resources", disable_resources, self.disable_resources),- ("wait_selector", wait_selector, self.wait_selector),- ("wait_selector_state", wait_selector_state, self.wait_selector_state),- ("network_idle", network_idle, self.network_idle),- ("load_dom", load_dom, self.load_dom),- ("solve_cloudflare", solve_cloudflare, self.solve_cloudflare),- ("selector_config", selector_config, self.selector_config),- ],- CamoufoxConfig,- _UNSET,- )+ params = _validate(kwargs, self, CamoufoxConfig)if self._closed: # pragma: no coverraise RuntimeError("Context manager has been closed")@@ -322,7 +223,7 @@ class StealthySession(StealthySessionMixin, SyncSession):raise RuntimeError(f"Failed to get response for {url}")if params.solve_cloudflare:- self._solve_cloudflare(page_info.page)+ self._cloudflare_solver(page_info.page)# Make sure the page is fully loaded after the captchaself._wait_for_page_stability(page_info.page, params.load_dom, params.network_idle)@@ -360,36 +261,7 @@ class StealthySession(StealthySessionMixin, SyncSession):class AsyncStealthySession(StealthySessionMixin, AsyncSession):"""A Stealthy session manager with page pooling."""- def __init__(- self,- max_pages: int = 1,- headless: bool = True, # noqa: F821- block_images: bool = False,- disable_resources: bool = False,- block_webrtc: bool = False,- allow_webgl: bool = True,- network_idle: bool = False,- load_dom: bool = True,- humanize: bool | float = True,- solve_cloudflare: bool = False,- wait: int | float = 0,- timeout: int | float = 30000,- page_action: Optional[Callable] = None,- wait_selector: Optional[str] = None,- init_script: Optional[str] = None,- addons: Optional[List[str]] = None,- wait_selector_state: SelectorWaitStates = "attached",- cookies: Optional[List[Dict]] = None,- google_search: bool = True,- extra_headers: Optional[Dict[str, str]] = None,- proxy: Optional[str | Dict[str, str]] = None,- os_randomize: bool = False,- disable_ads: bool = False,- geoip: bool = False,- user_data_dir: str = "",- selector_config: Optional[Dict] = None,- additional_args: Optional[Dict] = None,- ):+ def __init__(self, **kwargs: Unpack[CamoufoxSession]):"""A Browser session manager with page pooling:param headless: Run the browser in headless/hidden (default), or headful/visible mode.@@ -424,36 +296,8 @@ class AsyncStealthySession(StealthySessionMixin, AsyncSession)::param selector_config: The arguments that will be passed in the end while creating the final Selector's class.:param additional_args: Additional arguments to be passed to Camoufox as additional settings, and it takes higher priority than Scrapling's settings."""- self.__validate__(- wait=wait,- proxy=proxy,- geoip=geoip,- addons=addons,- timeout=timeout,- cookies=cookies,- headless=headless,- load_dom=load_dom,- humanize=humanize,- max_pages=max_pages,- disable_ads=disable_ads,- allow_webgl=allow_webgl,- page_action=page_action,- init_script=init_script,- network_idle=network_idle,- block_images=block_images,- block_webrtc=block_webrtc,- os_randomize=os_randomize,- wait_selector=wait_selector,- google_search=google_search,- extra_headers=extra_headers,- user_data_dir=user_data_dir,- additional_args=additional_args,- selector_config=selector_config,- solve_cloudflare=solve_cloudflare,- disable_resources=disable_resources,- wait_selector_state=wait_selector_state,- )- super().__init__(max_pages=self.max_pages)+ self.__validate__(**kwargs)+ super().__init__(max_pages=self._max_pages)async def __create__(self):"""Create a browser for this instance and context."""@@ -462,13 +306,13 @@ class AsyncStealthySession(StealthySessionMixin, AsyncSession):**self.launch_options)- if self.init_script: # pragma: no cover- await self.context.add_init_script(path=self.init_script)+ if self._init_script: # pragma: no cover+ await self.context.add_init_script(path=self._init_script)- if self.cookies:- await self.context.add_cookies(self.cookies) # pyright: ignore [reportArgumentType]+ if self._cookies:+ await self.context.add_cookies(self._cookies) # pyright: ignore [reportArgumentType]- async def _solve_cloudflare(self, page: async_Page): # pragma: no cover+ async def _cloudflare_solver(self, page: async_Page): # pragma: no cover"""Solve the cloudflare challenge displayed on the playwright page passed. The async version:param page: The async targeted page@@ -534,59 +378,28 @@ class AsyncStealthySession(StealthySessionMixin, AsyncSession):log.info("Cloudflare captcha is solved")return- async def fetch(- self,- url: str,- google_search: bool = _UNSET,- timeout: int | float = _UNSET,- wait: int | float = _UNSET,- page_action: Optional[Callable] = _UNSET,- extra_headers: Optional[Dict[str, str]] = _UNSET,- disable_resources: bool = _UNSET,- wait_selector: Optional[str] = _UNSET,- wait_selector_state: SelectorWaitStates = _UNSET,- network_idle: bool = _UNSET,- load_dom: bool = _UNSET,- solve_cloudflare: bool = _UNSET,- selector_config: Optional[Dict] = _UNSET,- ) -> Response:+ async def fetch(self, url: str, **kwargs: Unpack[CamoufoxFetchParams]) -> Response:"""Opens up the browser and do your request based on your chosen options.:param url: The Target url.- :param google_search: Enabled by default, Scrapling will set the referer header to be as if this request came from a Google search of this website's domain name.- :param timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30,000- :param wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the ` Response ` object.- :param page_action: Added for automation. A function that takes the `page` object and does the automation you need.- :param extra_headers: A dictionary of extra headers to add to the request. _The referer set by the `google_search` argument takes priority over the referer set here if used together._- :param disable_resources: Drop requests of unnecessary resources for a speed boost. It depends, but it made requests ~25% faster in my tests for some websites.- Requests dropped are of type `font`, `image`, `media`, `beacon`, `object`, `imageset`, `texttrack`, `websocket`, `csp_report`, and `stylesheet`.- This can help save your proxy usage but be careful with this option as it makes some websites never finish loading.- :param wait_selector: Wait for a specific CSS selector to be in a specific state.- :param wait_selector_state: The state to wait for the selector given with `wait_selector`. The default state is `attached`.- :param network_idle: Wait for the page until there are no network connections for at least 500 ms.- :param load_dom: Enabled by default, wait for all JavaScript on page(s) to fully load and execute.- :param solve_cloudflare: Solves all types of the Cloudflare's Turnstile/Interstitial challenges before returning the response to you.- :param selector_config: The arguments that will be passed in the end while creating the final Selector's class.+ :param kwargs: Additional keyword arguments including:+ - google_search: Enabled by default, Scrapling will set the referer header to be as if this request came from a Google search of this website's domain name.+ - timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30,000+ - wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the ` Response ` object.+ - page_action: Added for automation. A function that takes the `page` object and does the automation you need.+ - extra_headers: A dictionary of extra headers to add to the request. _The referer set by the `google_search` argument takes priority over the referer set here if used together._+ - disable_resources: Drop requests of unnecessary resources for a speed boost. It depends, but it made requests ~25% faster in my tests for some websites.+ Requests dropped are of type `font`, `image`, `media`, `beacon`, `object`, `imageset`, `texttrack`, `websocket`, `csp_report`, and `stylesheet`.+ This can help save your proxy usage but be careful with this option as it makes some websites never finish loading.+ - wait_selector: Wait for a specific CSS selector to be in a specific state.+ - wait_selector_state: The state to wait for the selector given with `wait_selector`. The default state is `attached`.+ - network_idle: Wait for the page until there are no network connections for at least 500 ms.+ - load_dom: Enabled by default, wait for all JavaScript on page(s) to fully load and execute.+ - solve_cloudflare: Solves all types of the Cloudflare's Turnstile/Interstitial challenges before returning the response to you.+ - selector_config: The arguments that will be passed in the end while creating the final Selector's class.:return: A `Response` object."""- params = _validate(- [- ("google_search", google_search, self.google_search),- ("timeout", timeout, self.timeout),- ("wait", wait, self.wait),- ("page_action", page_action, self.page_action),- ("extra_headers", extra_headers, self.extra_headers),- ("disable_resources", disable_resources, self.disable_resources),- ("wait_selector", wait_selector, self.wait_selector),- ("wait_selector_state", wait_selector_state, self.wait_selector_state),- ("network_idle", network_idle, self.network_idle),- ("load_dom", load_dom, self.load_dom),- ("solve_cloudflare", solve_cloudflare, self.solve_cloudflare),- ("selector_config", selector_config, self.selector_config),- ],- CamoufoxConfig,- _UNSET,- )+ params = _validate(kwargs, self, CamoufoxConfig)if self._closed: # pragma: no coverraise RuntimeError("Context manager has been closed")@@ -613,7 +426,7 @@ class AsyncStealthySession(StealthySessionMixin, AsyncSession):raise RuntimeError(f"Failed to get response for {url}")if params.solve_cloudflare:- await self._solve_cloudflare(page_info.page)+ await self._cloudflare_solver(page_info.page)# Make sure the page is fully loaded after the captchaawait self._wait_for_page_stability(page_info.page, params.load_dom, params.network_idle)diff --git a/scrapling/engines/_browsers/_controllers.py b/scrapling/engines/_browsers/_controllers.pyindex 9fc3490..9ab0e3c 100644--- a/scrapling/engines/_browsers/_controllers.py+++ b/scrapling/engines/_browsers/_controllers.py@@ -13,92 +13,55 @@ from patchright.sync_api import sync_playwright as sync_patchrightfrom patchright.async_api import async_playwright as async_patchrightfrom scrapling.core.utils import log+from scrapling.core._types import Unpack, TYPE_CHECKING+from ._types import PlaywrightSession, PlaywrightFetchParamsfrom ._base import SyncSession, AsyncSession, DynamicSessionMixinfrom ._validators import validate_fetch as _validate, PlaywrightConfig-from scrapling.core._types import (- Any,- Dict,- List,- Optional,- Callable,- TYPE_CHECKING,- SelectorWaitStates,-)-from scrapling.engines.toolbelt.convertor import (- Response,- ResponseFactory,-)+from scrapling.engines.toolbelt.convertor import Response, ResponseFactoryfrom scrapling.engines.toolbelt.fingerprints import generate_convincing_referer-_UNSET: Any = object()-class DynamicSession(DynamicSessionMixin, SyncSession):"""A Browser session manager with page pooling."""__slots__ = (- "max_pages",- "headless",- "hide_canvas",- "disable_webgl",- "real_chrome",- "stealth",- "google_search",- "proxy",- "locale",- "extra_headers",- "useragent",- "timeout",- "cookies",- "disable_resources",- "network_idle",- "load_dom",- "wait_selector",- "init_script",- "wait_selector_state",- "wait",+ "_max_pages",+ "_headless",+ "_hide_canvas",+ "_disable_webgl",+ "_real_chrome",+ "_stealth",+ "_google_search",+ "_proxy",+ "_locale",+ "_extra_headers",+ "_useragent",+ "_timeout",+ "_cookies",+ "_disable_resources",+ "_network_idle",+ "_load_dom",+ "_wait_selector",+ "_init_script",+ "_wait_selector_state",+ "_wait","playwright","browser","context","page_pool","_closed",- "selector_config",- "page_action",+ "_selector_config",+ "_page_action","launch_options","context_options",- "cdp_url",+ "_cdp_url","_headers_keys",+ "_extra_flags",+ "_additional_args",+ "_user_data_dir",)- def __init__(- self,- __max_pages: int = 1,- headless: bool = True,- google_search: bool = True,- hide_canvas: bool = False,- disable_webgl: bool = False,- real_chrome: bool = False,- stealth: bool = False,- wait: int | float = 0,- page_action: Optional[Callable] = None,- proxy: Optional[str | Dict[str, str]] = None,- locale: str = "en-US",- extra_headers: Optional[Dict[str, str]] = None,- useragent: Optional[str] = None,- cdp_url: Optional[str] = None,- timeout: int | float = 30000,- disable_resources: bool = False,- wait_selector: Optional[str] = None,- init_script: Optional[str] = None,- cookies: Optional[List[Dict]] = None,- network_idle: bool = False,- load_dom: bool = True,- wait_selector_state: SelectorWaitStates = "attached",- user_data_dir: str = "",- extra_flags: Optional[List[str]] = None,- selector_config: Optional[Dict] = None,- additional_args: Optional[Dict] = None,- ):+ def __init__(self, **kwargs: Unpack[PlaywrightSession]):"""A Browser session manager with page pooling, it's using a persistent browser Context by default with a temporary user profile directory.:param headless: Run the browser in headless/hidden (default), or headful/visible mode.@@ -129,105 +92,49 @@ class DynamicSession(DynamicSessionMixin, SyncSession)::param selector_config: The arguments that will be passed in the end while creating the final Selector's class.:param additional_args: Additional arguments to be passed to Playwright's context as additional settings, and it takes higher priority than Scrapling's settings."""- self.__validate__(- wait=wait,- proxy=proxy,- locale=locale,- timeout=timeout,- stealth=stealth,- cdp_url=cdp_url,- cookies=cookies,- load_dom=load_dom,- headless=headless,- useragent=useragent,- max_pages=__max_pages,- real_chrome=real_chrome,- page_action=page_action,- hide_canvas=hide_canvas,- init_script=init_script,- network_idle=network_idle,- user_data_dir=user_data_dir,- google_search=google_search,- extra_headers=extra_headers,- wait_selector=wait_selector,- disable_webgl=disable_webgl,- extra_flags=extra_flags,- selector_config=selector_config,- additional_args=additional_args,- disable_resources=disable_resources,- wait_selector_state=wait_selector_state,- )- super().__init__(max_pages=self.max_pages)+ self.__validate__(**kwargs)+ super().__init__(max_pages=self._max_pages)def __create__(self):"""Create a browser for this instance and context."""- sync_context = sync_patchright if self.stealth else sync_playwright+ sync_context = sync_patchright if self._stealth else sync_playwrightself.playwright: Playwright = sync_context().start() # pyright: ignore [reportAttributeAccessIssue]- if self.cdp_url: # pragma: no cover- self.context = self.playwright.chromium.connect_over_cdp(endpoint_url=self.cdp_url).new_context(+ if self._cdp_url: # pragma: no cover+ self.context = self.playwright.chromium.connect_over_cdp(endpoint_url=self._cdp_url).new_context(**self.context_options)else:self.context = self.playwright.chromium.launch_persistent_context(**self.launch_options)- if self.init_script: # pragma: no cover- self.context.add_init_script(path=self.init_script)-- if self.cookies: # pragma: no cover- self.context.add_cookies(self.cookies)-- def fetch(- self,- url: str,- google_search: bool = _UNSET,- timeout: int | float = _UNSET,- wait: int | float = _UNSET,- page_action: Optional[Callable] = _UNSET,- extra_headers: Optional[Dict[str, str]] = _UNSET,- disable_resources: bool = _UNSET,- wait_selector: Optional[str] = _UNSET,- wait_selector_state: SelectorWaitStates = _UNSET,- network_idle: bool = _UNSET,- load_dom: bool = _UNSET,- selector_config: Optional[Dict] = _UNSET,- ) -> Response:+ if self._init_script: # pragma: no cover+ self.context.add_init_script(path=self._init_script)++ if self._cookies: # pragma: no cover+ self.context.add_cookies(self._cookies)++ def fetch(self, url: str, **kwargs: Unpack[PlaywrightFetchParams]) -> Response:"""Opens up the browser and do your request based on your chosen options.:param url: The Target url.- :param google_search: Enabled by default, Scrapling will set the referer header to be as if this request came from a Google search of this website's domain name.- :param timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30,000- :param wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the ` Response ` object.- :param page_action: Added for automation. A function that takes the `page` object and does the automation you need.- :param extra_headers: A dictionary of extra headers to add to the request. _The referer set by the `google_search` argument takes priority over the referer set here if used together._- :param disable_resources: Drop requests of unnecessary resources for a speed boost. It depends, but it made requests ~25% faster in my tests for some websites.- Requests dropped are of type `font`, `image`, `media`, `beacon`, `object`, `imageset`, `texttrack`, `websocket`, `csp_report`, and `stylesheet`.- This can help save your proxy usage but be careful with this option as it makes some websites never finish loading.- :param wait_selector: Wait for a specific CSS selector to be in a specific state.- :param wait_selector_state: The state to wait for the selector given with `wait_selector`. The default state is `attached`.- :param network_idle: Wait for the page until there are no network connections for at least 500 ms.- :param load_dom: Enabled by default, wait for all JavaScript on page(s) to fully load and execute.- :param selector_config: The arguments that will be passed in the end while creating the final Selector's class.+ :param kwargs: Additional keyword arguments including:+ - google_search: Enabled by default, Scrapling will set the referer header to be as if this request came from a Google search of this website's domain name.+ - timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30,000+ - wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the ` Response ` object.+ - page_action: Added for automation. A function that takes the `page` object and does the automation you need.+ - extra_headers: A dictionary of extra headers to add to the request. _The referer set by the `google_search` argument takes priority over the referer set here if used together._+ - disable_resources: Drop requests of unnecessary resources for a speed boost. It depends, but it made requests ~25% faster in my tests for some websites.+ Requests dropped are of type `font`, `image`, `media`, `beacon`, `object`, `imageset`, `texttrack`, `websocket`, `csp_report`, and `stylesheet`.+ This can help save your proxy usage but be careful with this option as it makes some websites never finish loading.+ - wait_selector: Wait for a specific CSS selector to be in a specific state.+ - wait_selector_state: The state to wait for the selector given with `wait_selector`. The default state is `attached`.+ - network_idle: Wait for the page until there are no network connections for at least 500 ms.+ - load_dom: Enabled by default, wait for all JavaScript on page(s) to fully load and execute.+ - selector_config: The arguments that will be passed in the end while creating the final Selector's class.:return: A `Response` object."""- params = _validate(- [- ("google_search", google_search, self.google_search),- ("timeout", timeout, self.timeout),- ("wait", wait, self.wait),- ("page_action", page_action, self.page_action),- ("extra_headers", extra_headers, self.extra_headers),- ("disable_resources", disable_resources, self.disable_resources),- ("wait_selector", wait_selector, self.wait_selector),- ("wait_selector_state", wait_selector_state, self.wait_selector_state),- ("network_idle", network_idle, self.network_idle),- ("load_dom", load_dom, self.load_dom),- ("selector_config", selector_config, self.selector_config),- ],- PlaywrightConfig,- _UNSET,- )+ params = _validate(kwargs, self, PlaywrightConfig)if self._closed: # pragma: no coverraise RuntimeError("Context manager has been closed")@@ -285,35 +192,7 @@ class DynamicSession(DynamicSessionMixin, SyncSession):class AsyncDynamicSession(DynamicSessionMixin, AsyncSession):"""An async Browser session manager with page pooling, it's using a persistent browser Context by default with a temporary user profile directory."""- def __init__(- self,- max_pages: int = 1,- headless: bool = True,- google_search: bool = True,- hide_canvas: bool = False,- disable_webgl: bool = False,- real_chrome: bool = False,- stealth: bool = False,- wait: int | float = 0,- page_action: Optional[Callable] = None,- proxy: Optional[str | Dict[str, str]] = None,- locale: str = "en-US",- extra_headers: Optional[Dict[str, str]] = None,- useragent: Optional[str] = None,- cdp_url: Optional[str] = None,- timeout: int | float = 30000,- disable_resources: bool = False,- wait_selector: Optional[str] = None,- init_script: Optional[str] = None,- cookies: Optional[List[Dict]] = None,- network_idle: bool = False,- load_dom: bool = True,- wait_selector_state: SelectorWaitStates = "attached",- user_data_dir: str = "",- extra_flags: Optional[List[str]] = None,- selector_config: Optional[Dict] = None,- additional_args: Optional[Dict] = None,- ):+ def __init__(self, **kwargs: Unpack[PlaywrightSession]):"""A Browser session manager with page pooling:param headless: Run the browser in headless/hidden (default), or headful/visible mode.@@ -345,107 +224,50 @@ class AsyncDynamicSession(DynamicSessionMixin, AsyncSession)::param selector_config: The arguments that will be passed in the end while creating the final Selector's class.:param additional_args: Additional arguments to be passed to Playwright's context as additional settings, and it takes higher priority than Scrapling's settings."""-- self.__validate__(- wait=wait,- proxy=proxy,- locale=locale,- timeout=timeout,- stealth=stealth,- cdp_url=cdp_url,- cookies=cookies,- load_dom=load_dom,- headless=headless,- useragent=useragent,- max_pages=max_pages,- real_chrome=real_chrome,- page_action=page_action,- hide_canvas=hide_canvas,- init_script=init_script,- network_idle=network_idle,- user_data_dir=user_data_dir,- google_search=google_search,- extra_headers=extra_headers,- wait_selector=wait_selector,- disable_webgl=disable_webgl,- extra_flags=extra_flags,- selector_config=selector_config,- additional_args=additional_args,- disable_resources=disable_resources,- wait_selector_state=wait_selector_state,- )- super().__init__(max_pages=self.max_pages)+ self.__validate__(**kwargs)+ super().__init__(max_pages=self._max_pages)async def __create__(self):"""Create a browser for this instance and context."""- async_context = async_patchright if self.stealth else async_playwright+ async_context = async_patchright if self._stealth else async_playwrightself.playwright: AsyncPlaywright = await async_context().start() # pyright: ignore [reportAttributeAccessIssue]- if self.cdp_url:- browser = await self.playwright.chromium.connect_over_cdp(endpoint_url=self.cdp_url)+ if self._cdp_url:+ browser = await self.playwright.chromium.connect_over_cdp(endpoint_url=self._cdp_url)self.context: AsyncBrowserContext = await browser.new_context(**self.context_options)else:self.context: AsyncBrowserContext = await self.playwright.chromium.launch_persistent_context(**self.launch_options)- if self.init_script: # pragma: no cover- await self.context.add_init_script(path=self.init_script)-- if self.cookies:- await self.context.add_cookies(self.cookies) # pyright: ignore-- async def fetch(- self,- url: str,- google_search: bool = _UNSET,- timeout: int | float = _UNSET,- wait: int | float = _UNSET,- page_action: Optional[Callable] = _UNSET,- extra_headers: Optional[Dict[str, str]] = _UNSET,- disable_resources: bool = _UNSET,- wait_selector: Optional[str] = _UNSET,- wait_selector_state: SelectorWaitStates = _UNSET,- network_idle: bool = _UNSET,- load_dom: bool = _UNSET,- selector_config: Optional[Dict] = _UNSET,- ) -> Response:+ if self._init_script: # pragma: no cover+ await self.context.add_init_script(path=self._init_script)++ if self._cookies:+ await self.context.add_cookies(self._cookies) # pyright: ignore++ async def fetch(self, url: str, **kwargs: Unpack[PlaywrightFetchParams]) -> Response:"""Opens up the browser and do your request based on your chosen options.:param url: The Target url.- :param google_search: Enabled by default, Scrapling will set the referer header to be as if this request came from a Google search of this website's domain name.- :param timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30,000- :param wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the ` Response ` object.- :param page_action: Added for automation. A function that takes the `page` object and does the automation you need.- :param extra_headers: A dictionary of extra headers to add to the request. _The referer set by the `google_search` argument takes priority over the referer set here if used together._- :param disable_resources: Drop requests of unnecessary resources for a speed boost. It depends, but it made requests ~25% faster in my tests for some websites.- Requests dropped are of type `font`, `image`, `media`, `beacon`, `object`, `imageset`, `texttrack`, `websocket`, `csp_report`, and `stylesheet`.- This can help save your proxy usage but be careful with this option as it makes some websites never finish loading.- :param wait_selector: Wait for a specific CSS selector to be in a specific state.- :param wait_selector_state: The state to wait for the selector given with `wait_selector`. The default state is `attached`.- :param network_idle: Wait for the page until there are no network connections for at least 500 ms.- :param load_dom: Enabled by default, wait for all JavaScript on page(s) to fully load and execute.- :param selector_config: The arguments that will be passed in the end while creating the final Selector's class.+ :param kwargs: Additional keyword arguments including:+ - google_search: Enabled by default, Scrapling will set the referer header to be as if this request came from a Google search of this website's domain name.+ - timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30,000+ - wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the ` Response ` object.+ - page_action: Added for automation. A function that takes the `page` object and does the automation you need.+ - extra_headers: A dictionary of extra headers to add to the request. _The referer set by the `google_search` argument takes priority over the referer set here if used together._+ - disable_resources: Drop requests of unnecessary resources for a speed boost. It depends, but it made requests ~25% faster in my tests for some websites.+ Requests dropped are of type `font`, `image`, `media`, `beacon`, `object`, `imageset`, `texttrack`, `websocket`, `csp_report`, and `stylesheet`.+ This can help save your proxy usage but be careful with this option as it makes some websites never finish loading.+ - wait_selector: Wait for a specific CSS selector to be in a specific state.+ - wait_selector_state: The state to wait for the selector given with `wait_selector`. The default state is `attached`.+ - network_idle: Wait for the page until there are no network connections for at least 500 ms.+ - load_dom: Enabled by default, wait for all JavaScript on page(s) to fully load and execute.+ - selector_config: The arguments that will be passed in the end while creating the final Selector's class.:return: A `Response` object."""- params = _validate(- [- ("google_search", google_search, self.google_search),- ("timeout", timeout, self.timeout),- ("wait", wait, self.wait),- ("page_action", page_action, self.page_action),- ("extra_headers", extra_headers, self.extra_headers),- ("disable_resources", disable_resources, self.disable_resources),- ("wait_selector", wait_selector, self.wait_selector),- ("wait_selector_state", wait_selector_state, self.wait_selector_state),- ("network_idle", network_idle, self.network_idle),- ("load_dom", load_dom, self.load_dom),- ("selector_config", selector_config, self.selector_config),- ],- PlaywrightConfig,- _UNSET,- )+ params = _validate(kwargs, self, PlaywrightConfig)if self._closed: # pragma: no coverraise RuntimeError("Context manager has been closed")diff --git a/scrapling/engines/_browsers/_page.py b/scrapling/engines/_browsers/_page.pyindex 821fbbb..655d3d1 100644--- a/scrapling/engines/_browsers/_page.py+++ b/scrapling/engines/_browsers/_page.py@@ -1,20 +1,21 @@from threading import RLockfrom dataclasses import dataclass-from playwright.sync_api import Page as SyncPage-from playwright.async_api import Page as AsyncPage+from playwright.sync_api._generated import Page as SyncPage+from playwright.async_api._generated import Page as AsyncPage-from scrapling.core._types import Optional, List, Literal+from scrapling.core._types import Optional, List, Literal, overload, TypeVar, Generic, castPageState = Literal["ready", "busy", "error"] # States that a page can be in+PageType = TypeVar("PageType", SyncPage, AsyncPage)@dataclass-class PageInfo:+class PageInfo(Generic[PageType]):"""Information about the page and its current state"""__slots__ = ("page", "state", "url")- page: SyncPage | AsyncPage+ page: PageTypestate: PageStateurl: Optional[str]@@ -44,16 +45,26 @@ class PagePool:def __init__(self, max_pages: int = 5):self.max_pages = max_pages- self.pages: List[PageInfo] = []+ self.pages: List[PageInfo[SyncPage] | PageInfo[AsyncPage]] = []self._lock = RLock()- def add_page(self, page: SyncPage | AsyncPage) -> PageInfo:+ @overload+ def add_page(self, page: SyncPage) -> PageInfo[SyncPage]: ...++ @overload+ def add_page(self, page: AsyncPage) -> PageInfo[AsyncPage]: ...++ def add_page(self, page: SyncPage | AsyncPage) -> PageInfo[SyncPage] | PageInfo[AsyncPage]:"""Add a new page to the pool"""with self._lock:if len(self.pages) >= self.max_pages:raise RuntimeError(f"Maximum page limit ({self.max_pages}) reached")- page_info = PageInfo(page, "ready", "")+ if isinstance(page, AsyncPage):+ page_info = cast(PageInfo[AsyncPage], PageInfo(page, "ready", ""))+ else:+ page_info = cast(PageInfo[SyncPage], PageInfo(page, "ready", ""))+self.pages.append(page_info)return page_infodiff --git a/scrapling/engines/_browsers/_types.py b/scrapling/engines/_browsers/_types.pyindex 0dd0c91..d1edca5 100644--- a/scrapling/engines/_browsers/_types.py+++ b/scrapling/engines/_browsers/_types.py@@ -10,8 +10,11 @@ from scrapling.core._types import (Tuple,Mapping,Optional,+ Callable,+ Iterable,TypedDict,TypeAlias,+ SelectorWaitStates,TYPE_CHECKING,)@@ -49,7 +52,69 @@ if TYPE_CHECKING: # pragma: no coverdata: Optional[Dict | str]json: Optional[Dict | List]+ # Types for browser session+ class BrowserSession(TypedDict, total=False):+ max_pages: int+ headless: bool+ disable_resources: bool+ network_idle: bool+ load_dom: bool+ wait_selector: Optional[str]+ wait_selector_state: SelectorWaitStates+ cookies: Optional[Iterable[Dict]]+ google_search: bool+ wait: int | float+ page_action: Optional[Callable]+ proxy: Optional[str | Dict[str, str] | Tuple]+ extra_headers: Optional[Dict[str, str]]+ timeout: int | float+ init_script: Optional[str]+ user_data_dir: str+ selector_config: Optional[Dict]+ additional_args: Optional[Dict]++ class PlaywrightSession(BrowserSession, total=False):+ cdp_url: Optional[str]+ hide_canvas: bool+ disable_webgl: bool+ real_chrome: bool+ stealth: bool+ locale: str+ useragent: Optional[str]+ extra_flags: Optional[List[str]]++ class PlaywrightFetchParams(TypedDict, total=False):+ google_search: bool+ timeout: int | float+ wait: int | float+ page_action: Optional[Callable]+ extra_headers: Optional[Dict[str, str]]+ disable_resources: bool+ wait_selector: Optional[str]+ wait_selector_state: SelectorWaitStates+ network_idle: bool+ load_dom: bool+ selector_config: Optional[Dict]++ class CamoufoxSession(BrowserSession, total=False):+ block_images: bool+ block_webrtc: bool+ allow_webgl: bool+ humanize: bool | float+ solve_cloudflare: bool+ addons: Optional[List[str]]+ os_randomize: bool+ disable_ads: bool+ geoip: bool++ class CamoufoxFetchParams(PlaywrightFetchParams, total=False):+ solve_cloudflare: bool+else: # pragma: no coverRequestsSession = TypedDictGetRequestParams = TypedDictDataRequestParams = TypedDict+ PlaywrightSession = TypedDict+ PlaywrightFetchParams = TypedDict+ CamoufoxSession = TypedDict+ CamoufoxFetchParams = TypedDictdiff --git a/scrapling/engines/_browsers/_validators.py b/scrapling/engines/_browsers/_validators.pyindex 5442102..a867a37 100644--- a/scrapling/engines/_browsers/_validators.py+++ b/scrapling/engines/_browsers/_validators.py@@ -7,6 +7,7 @@ from dataclasses import dataclass, fieldsfrom msgspec import Struct, Meta, convert, ValidationErrorfrom scrapling.core._types import (+ Any,Dict,List,Tuple,@@ -17,6 +18,7 @@ from scrapling.core._types import (overload,)from scrapling.engines.toolbelt.navigation import construct_proxy_dict+from scrapling.engines._browsers._types import PlaywrightFetchParams, CamoufoxFetchParams# Custom validators for msgspec@@ -194,16 +196,24 @@ class _fetch_params:def validate_fetch(- params: List[Tuple], model: type[PlaywrightConfig] | type[CamoufoxConfig], sentinel=None+ method_kwargs: Dict | PlaywrightFetchParams | CamoufoxFetchParams,+ session: Any,+ model: type[PlaywrightConfig] | type[CamoufoxConfig],) -> _fetch_params: # pragma: no coverresult = {}overrides = {}- for arg, request_value, session_value in params:- if request_value is not sentinel:- overrides[arg] = request_value+ # Get all field names that _fetch_params needs+ fetch_param_fields = {f.name for f in fields(_fetch_params)}++ for key in fetch_param_fields:+ if key in method_kwargs:+ overrides[key] = method_kwargs[key]else:- result[arg] = session_value+ # Check for underscore-prefixed attribute (private)+ attr_name = f"_{key}"+ if hasattr(session, attr_name):+ result[key] = getattr(session, attr_name)if overrides:validated_config = validate(overrides, model)