🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev
Python aiai-scrapingautomationcrawlercrawlingcrawling-pythondatadata-extractionmcpmcp-serverplaywrightpythonscrapingselectorsstealthweb-scraperweb-scrapingweb-scraping-pythonwebscrapingxpath
perf: Improving all browser-based fetches by ~15%
Karim shoair82b09620b28f
diff --git a/scrapling/engines/_browsers/_base.py b/scrapling/engines/_browsers/_base.pyindex 703a915..69681ee 100644--- a/scrapling/engines/_browsers/_base.py+++ b/scrapling/engines/_browsers/_base.py@@ -60,11 +60,6 @@ class SyncSession:return self.page_pool.add_page(page)- @staticmethod- def _get_with_precedence(request_value: Any, session_value: Any, sentinel_value: object) -> Any:- """Get value with request-level priority over session-level"""- return request_value if request_value is not sentinel_value else session_value-def get_pool_stats(self) -> Dict[str, int]:"""Get statistics about the current page pool"""return {diff --git a/scrapling/engines/_browsers/_camoufox.py b/scrapling/engines/_browsers/_camoufox.pyindex 18ec633..5f0e795 100644--- a/scrapling/engines/_browsers/_camoufox.py+++ b/scrapling/engines/_browsers/_camoufox.py@@ -16,7 +16,7 @@ from playwright.async_api import ()from playwright._impl._errors import Error as PlaywrightError-from ._validators import validate, CamoufoxConfig+from ._validators import validate_fetch as _validatefrom ._base import SyncSession, AsyncSession, StealthySessionMixinfrom scrapling.core.utils import logfrom scrapling.core._types import (@@ -297,23 +297,22 @@ class StealthySession(StealthySessionMixin, SyncSession)::param selector_config: The arguments that will be passed in the end while creating the final Selector's class.:return: A `Response` object."""- # Validate all resolved parameters- params = validate(- dict(- google_search=self._get_with_precedence(google_search, self.google_search, _UNSET),- timeout=self._get_with_precedence(timeout, self.timeout, _UNSET),- wait=self._get_with_precedence(wait, self.wait, _UNSET),- page_action=self._get_with_precedence(page_action, self.page_action, _UNSET),- extra_headers=self._get_with_precedence(extra_headers, self.extra_headers, _UNSET),- disable_resources=self._get_with_precedence(disable_resources, self.disable_resources, _UNSET),- wait_selector=self._get_with_precedence(wait_selector, self.wait_selector, _UNSET),- wait_selector_state=self._get_with_precedence(wait_selector_state, self.wait_selector_state, _UNSET),- network_idle=self._get_with_precedence(network_idle, self.network_idle, _UNSET),- load_dom=self._get_with_precedence(load_dom, self.load_dom, _UNSET),- solve_cloudflare=self._get_with_precedence(solve_cloudflare, self.solve_cloudflare, _UNSET),- selector_config=self._get_with_precedence(selector_config, self.selector_config, _UNSET),- ),- CamoufoxConfig,+ params = _validate(+ [+ ("google_search", google_search, self.google_search),+ ("timeout", timeout, self.timeout),+ ("wait", wait, self.wait),+ ("page_action", page_action, self.page_action),+ ("extra_headers", extra_headers, self.extra_headers),+ ("disable_resources", disable_resources, self.disable_resources),+ ("wait_selector", wait_selector, self.wait_selector),+ ("wait_selector_state", wait_selector_state, self.wait_selector_state),+ ("network_idle", network_idle, self.network_idle),+ ("load_dom", load_dom, self.load_dom),+ ("solve_cloudflare", solve_cloudflare, self.solve_cloudflare),+ ("selector_config", selector_config, self.selector_config),+ ],+ _UNSET,)if self._closed: # pragma: no cover@@ -617,22 +616,22 @@ class AsyncStealthySession(StealthySessionMixin, AsyncSession)::param selector_config: The arguments that will be passed in the end while creating the final Selector's class.:return: A `Response` object."""- params = validate(- dict(- google_search=self._get_with_precedence(google_search, self.google_search, _UNSET),- timeout=self._get_with_precedence(timeout, self.timeout, _UNSET),- wait=self._get_with_precedence(wait, self.wait, _UNSET),- page_action=self._get_with_precedence(page_action, self.page_action, _UNSET),- extra_headers=self._get_with_precedence(extra_headers, self.extra_headers, _UNSET),- disable_resources=self._get_with_precedence(disable_resources, self.disable_resources, _UNSET),- wait_selector=self._get_with_precedence(wait_selector, self.wait_selector, _UNSET),- wait_selector_state=self._get_with_precedence(wait_selector_state, self.wait_selector_state, _UNSET),- network_idle=self._get_with_precedence(network_idle, self.network_idle, _UNSET),- load_dom=self._get_with_precedence(load_dom, self.load_dom, _UNSET),- solve_cloudflare=self._get_with_precedence(solve_cloudflare, self.solve_cloudflare, _UNSET),- selector_config=self._get_with_precedence(selector_config, self.selector_config, _UNSET),- ),- CamoufoxConfig,+ params = _validate(+ [+ ("google_search", google_search, self.google_search),+ ("timeout", timeout, self.timeout),+ ("wait", wait, self.wait),+ ("page_action", page_action, self.page_action),+ ("extra_headers", extra_headers, self.extra_headers),+ ("disable_resources", disable_resources, self.disable_resources),+ ("wait_selector", wait_selector, self.wait_selector),+ ("wait_selector_state", wait_selector_state, self.wait_selector_state),+ ("network_idle", network_idle, self.network_idle),+ ("load_dom", load_dom, self.load_dom),+ ("solve_cloudflare", solve_cloudflare, self.solve_cloudflare),+ ("selector_config", selector_config, self.selector_config),+ ],+ _UNSET,)if self._closed: # pragma: no coverdiff --git a/scrapling/engines/_browsers/_controllers.py b/scrapling/engines/_browsers/_controllers.pyindex 19362e7..b4b6276 100644--- a/scrapling/engines/_browsers/_controllers.py+++ b/scrapling/engines/_browsers/_controllers.py@@ -16,7 +16,7 @@ from patchright.async_api import async_playwright as async_patchrightfrom scrapling.core.utils import logfrom ._base import SyncSession, AsyncSession, DynamicSessionMixin-from ._validators import validate, PlaywrightConfig+from ._validators import validate_fetch as _validatefrom scrapling.core._types import (Dict,List,@@ -224,22 +224,21 @@ class DynamicSession(DynamicSessionMixin, SyncSession)::param selector_config: The arguments that will be passed in the end while creating the final Selector's class.:return: A `Response` object."""- # Validate all resolved parameters- params = validate(- dict(- google_search=self._get_with_precedence(google_search, self.google_search, _UNSET),- timeout=self._get_with_precedence(timeout, self.timeout, _UNSET),- wait=self._get_with_precedence(wait, self.wait, _UNSET),- page_action=self._get_with_precedence(page_action, self.page_action, _UNSET),- extra_headers=self._get_with_precedence(extra_headers, self.extra_headers, _UNSET),- disable_resources=self._get_with_precedence(disable_resources, self.disable_resources, _UNSET),- wait_selector=self._get_with_precedence(wait_selector, self.wait_selector, _UNSET),- wait_selector_state=self._get_with_precedence(wait_selector_state, self.wait_selector_state, _UNSET),- network_idle=self._get_with_precedence(network_idle, self.network_idle, _UNSET),- load_dom=self._get_with_precedence(load_dom, self.load_dom, _UNSET),- selector_config=self._get_with_precedence(selector_config, self.selector_config, _UNSET),- ),- PlaywrightConfig,+ params = _validate(+ [+ ("google_search", google_search, self.google_search),+ ("timeout", timeout, self.timeout),+ ("wait", wait, self.wait),+ ("page_action", page_action, self.page_action),+ ("extra_headers", extra_headers, self.extra_headers),+ ("disable_resources", disable_resources, self.disable_resources),+ ("wait_selector", wait_selector, self.wait_selector),+ ("wait_selector_state", wait_selector_state, self.wait_selector_state),+ ("network_idle", network_idle, self.network_idle),+ ("load_dom", load_dom, self.load_dom),+ ("selector_config", selector_config, self.selector_config),+ ],+ _UNSET,)if self._closed: # pragma: no cover@@ -471,22 +470,21 @@ class AsyncDynamicSession(DynamicSessionMixin, AsyncSession)::param selector_config: The arguments that will be passed in the end while creating the final Selector's class.:return: A `Response` object."""- # Validate all resolved parameters- params = validate(- dict(- google_search=self._get_with_precedence(google_search, self.google_search, _UNSET),- timeout=self._get_with_precedence(timeout, self.timeout, _UNSET),- wait=self._get_with_precedence(wait, self.wait, _UNSET),- page_action=self._get_with_precedence(page_action, self.page_action, _UNSET),- extra_headers=self._get_with_precedence(extra_headers, self.extra_headers, _UNSET),- disable_resources=self._get_with_precedence(disable_resources, self.disable_resources, _UNSET),- wait_selector=self._get_with_precedence(wait_selector, self.wait_selector, _UNSET),- wait_selector_state=self._get_with_precedence(wait_selector_state, self.wait_selector_state, _UNSET),- network_idle=self._get_with_precedence(network_idle, self.network_idle, _UNSET),- load_dom=self._get_with_precedence(load_dom, self.load_dom, _UNSET),- selector_config=self._get_with_precedence(selector_config, self.selector_config, _UNSET),- ),- PlaywrightConfig,+ params = _validate(+ [+ ("google_search", google_search, self.google_search),+ ("timeout", timeout, self.timeout),+ ("wait", wait, self.wait),+ ("page_action", page_action, self.page_action),+ ("extra_headers", extra_headers, self.extra_headers),+ ("disable_resources", disable_resources, self.disable_resources),+ ("wait_selector", wait_selector, self.wait_selector),+ ("wait_selector_state", wait_selector_state, self.wait_selector_state),+ ("network_idle", network_idle, self.network_idle),+ ("load_dom", load_dom, self.load_dom),+ ("selector_config", selector_config, self.selector_config),+ ],+ _UNSET,)if self._closed: # pragma: no coverdiff --git a/scrapling/engines/_browsers/_validators.py b/scrapling/engines/_browsers/_validators.pyindex ca64942..a2d7c60 100644--- a/scrapling/engines/_browsers/_validators.py+++ b/scrapling/engines/_browsers/_validators.py@@ -1,21 +1,69 @@-from msgspec import Struct, convert, ValidationError-from urllib.parse import urlparsefrom pathlib import Path+from typing import Annotated+from dataclasses import dataclass+from urllib.parse import urlparse++from msgspec import Struct, Meta, convert, ValidationErrorfrom scrapling.core._types import (- Optional,Dict,- Callable,List,+ Tuple,+ Optional,+ Callable,SelectorWaitStates,)from scrapling.engines.toolbelt.navigation import construct_proxy_dict+# Custom validators for msgspec+def _validate_file_path(value: str):+ """Fast file path validation"""+ path = Path(value)+ if not path.exists():+ raise ValueError(f"Init script path not found: {value}")+ if not path.is_file():+ raise ValueError(f"Init script is not a file: {value}")+ if not path.is_absolute():+ raise ValueError(f"Init script is not a absolute path: {value}")+++def _validate_addon_path(value: str):+ """Fast addon path validation"""+ path = Path(value)+ if not path.exists():+ raise FileNotFoundError(f"Addon path not found: {value}")+ if not path.is_dir():+ raise ValueError(f"Addon path must be a directory of the extracted addon: {value}")+++def _validate_cdp_url(cdp_url: str):+ """Fast CDP URL validation"""+ try:+ # Check the scheme+ if not cdp_url.startswith(("ws://", "wss://")):+ raise ValueError("CDP URL must use 'ws://' or 'wss://' scheme")++ # Validate hostname and port+ if not urlparse(cdp_url).netloc:+ raise ValueError("Invalid hostname for the CDP URL")++ except AttributeError as e:+ raise ValueError(f"Malformed CDP URL: {cdp_url}: {str(e)}")++ except Exception as e:+ raise ValueError(f"Invalid CDP URL '{cdp_url}': {str(e)}")+++# Type aliases for cleaner annotations+PagesCount = Annotated[int, Meta(ge=1, le=50)]+Seconds = Annotated[int, float, Meta(ge=0)]++class PlaywrightConfig(Struct, kw_only=True, frozen=False):"""Configuration struct for validation"""- max_pages: int = 1+ max_pages: PagesCount = 1cdp_url: Optional[str] = Noneheadless: bool = Truegoogle_search: bool = True@@ -23,13 +71,13 @@ class PlaywrightConfig(Struct, kw_only=True, frozen=False):disable_webgl: bool = Falsereal_chrome: bool = Falsestealth: bool = False- wait: int | float = 0+ wait: Seconds = 0page_action: Optional[Callable] = Noneproxy: Optional[str | Dict[str, str]] = None # The default value for proxy in Playwright's source is `None`locale: str = "en-US"extra_headers: Optional[Dict[str, str]] = Noneuseragent: Optional[str] = None- timeout: int | float = 30000+ timeout: Seconds = 30000init_script: Optional[str] = Nonedisable_resources: bool = Falsewait_selector: Optional[str] = None@@ -41,52 +89,26 @@ class PlaywrightConfig(Struct, kw_only=True, frozen=False):def __post_init__(self):"""Custom validation after msgspec validation"""- if self.max_pages < 1 or self.max_pages > 50:- raise ValueError("max_pages must be between 1 and 50")- if self.timeout < 0:- raise ValueError("timeout must be >= 0")if self.page_action and not callable(self.page_action):raise TypeError(f"page_action must be callable, got {type(self.page_action).__name__}")if self.proxy:self.proxy = construct_proxy_dict(self.proxy, as_tuple=True)if self.cdp_url:- self.__validate_cdp(self.cdp_url)+ _validate_cdp_url(self.cdp_url)+if not self.cookies:self.cookies = []if not self.selector_config:self.selector_config = {}if self.init_script is not None:- script_path = Path(self.init_script)- if not script_path.exists():- raise ValueError("Init script path not found")- elif not script_path.is_file():- raise ValueError("Init script is not a file")- elif not script_path.is_absolute():- raise ValueError("Init script is not a absolute path")-- @staticmethod- def __validate_cdp(cdp_url):- try:- # Check the scheme- if not cdp_url.startswith(("ws://", "wss://")):- raise ValueError("CDP URL must use 'ws://' or 'wss://' scheme")-- # Validate hostname and port- if not urlparse(cdp_url).netloc:- raise ValueError("Invalid hostname for the CDP URL")-- except AttributeError as e:- raise ValueError(f"Malformed CDP URL: {cdp_url}: {str(e)}")-- except Exception as e:- raise ValueError(f"Invalid CDP URL '{cdp_url}': {str(e)}")+ _validate_file_path(self.init_script)class CamoufoxConfig(Struct, kw_only=True, frozen=False):"""Configuration struct for validation"""- max_pages: int = 1+ max_pages: PagesCount = 1headless: bool = True # noqa: F821block_images: bool = Falsedisable_resources: bool = False@@ -96,8 +118,8 @@ class CamoufoxConfig(Struct, kw_only=True, frozen=False):load_dom: bool = Truehumanize: bool | float = Truesolve_cloudflare: bool = False- wait: int | float = 0- timeout: int | float = 30000+ wait: Seconds = 0+ timeout: Seconds = 30000init_script: Optional[str] = Nonepage_action: Optional[Callable] = Nonewait_selector: Optional[str] = None@@ -115,38 +137,23 @@ class CamoufoxConfig(Struct, kw_only=True, frozen=False):def __post_init__(self):"""Custom validation after msgspec validation"""- if self.max_pages < 1 or self.max_pages > 50:- raise ValueError("max_pages must be between 1 and 50")- if self.timeout < 0:- raise ValueError("timeout must be >= 0")if self.page_action and not callable(self.page_action):raise TypeError(f"page_action must be callable, got {type(self.page_action).__name__}")if self.proxy:self.proxy = construct_proxy_dict(self.proxy, as_tuple=True)- if not self.addons:- self.addons = []- else:+ if self.addons and isinstance(self.addons, list):for addon in self.addons:- addon_path = Path(addon)- if not addon_path.exists():- raise FileNotFoundError(f"Addon's path not found: {addon}")- elif not addon_path.is_dir():- raise ValueError(- f"Addon's path is not a folder, you need to pass a folder of the extracted addon: {addon}"- )+ _validate_addon_path(addon)+ else:+ self.addons = []if self.init_script is not None:- script_path = Path(self.init_script)- if not script_path.exists():- raise ValueError("Init script path not found")- elif not script_path.is_file():- raise ValueError("Init script is not a file")- elif not script_path.is_absolute():- raise ValueError("Init script is not a absolute path")+ _validate_file_path(self.init_script)if not self.cookies:self.cookies = []+ # Cloudflare timeout adjustmentif self.solve_cloudflare and self.timeout < 60_000:self.timeout = 60_000if not self.selector_config:@@ -155,10 +162,68 @@ class CamoufoxConfig(Struct, kw_only=True, frozen=False):self.additional_args = {}-def validate(params, model):+# Code parts to validate `fetch` in the least possible numbers of lines overall+class FetchConfig(Struct, kw_only=True):+ """Configuration struct for `fetch` calls validation"""++ google_search: bool = True+ timeout: Seconds = 30000+ wait: Seconds = 0+ page_action: Optional[Callable] = None+ extra_headers: Optional[Dict[str, str]] = None+ disable_resources: bool = False+ wait_selector: Optional[str] = None+ wait_selector_state: SelectorWaitStates = "attached"+ network_idle: bool = False+ load_dom: bool = True+ solve_cloudflare: bool = False+ selector_config: Optional[Dict] = {}++ def to_dict(self):+ return {f: getattr(self, f) for f in self.__struct_fields__}+++@dataclass+class _fetch_params:+ """A dataclass of all parameters used by `fetch` calls"""++ google_search: bool+ timeout: Seconds+ wait: Seconds+ page_action: Optional[Callable]+ extra_headers: Optional[Dict[str, str]]+ disable_resources: bool+ wait_selector: Optional[str]+ wait_selector_state: SelectorWaitStates+ network_idle: bool+ load_dom: bool+ solve_cloudflare: bool+ selector_config: Optional[Dict]+++def validate_fetch(params: List[Tuple], sentinel=None) -> _fetch_params:+ result = {}+ overrides = {}++ for arg, request_value, session_value in params:+ if request_value is not sentinel:+ overrides[arg] = request_value+ else:+ result[arg] = session_value++ if overrides:+ overrides = validate(overrides, FetchConfig).to_dict()+ overrides.update(result)+ return _fetch_params(**overrides)++ if not result.get("solve_cloudflare"):+ result["solve_cloudflare"] = False++ return _fetch_params(**result)+++def validate(params: Dict, model) -> PlaywrightConfig | CamoufoxConfig | FetchConfig:try:- config = convert(params, model)+ return convert(params, model)except ValidationError as e:- raise TypeError(f"Invalid argument type: {e}")-- return config+ raise TypeError(f"Invalid argument type: {e}") from e