🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev
Python aiai-scrapingautomationcrawlercrawlingcrawling-pythondatadata-extractionmcpmcp-serverplaywrightpythonscrapingselectorsstealthweb-scraperweb-scrapingweb-scraping-pythonwebscrapingxpath
refactor(validation): All browser based fetchers are ~8-15% faster now in most cases
Karim shoaird4a250766685
diff --git a/scrapling/engines/_browsers/_camoufox.py b/scrapling/engines/_browsers/_camoufox.pyindex ac66870..c0b2f2c 100644--- a/scrapling/engines/_browsers/_camoufox.py+++ b/scrapling/engines/_browsers/_camoufox.py@@ -13,9 +13,8 @@ from playwright.async_api import (Playwright as AsyncPlaywright,BrowserContext as AsyncBrowserContext,)-from playwright._impl._errors import Error as PlaywrightError-from ._validators import validate_fetch as _validate+from ._validators import validate_fetch as _validate, CamoufoxConfigfrom ._base import SyncSession, AsyncSession, StealthySessionMixinfrom scrapling.core.utils import logfrom scrapling.core._types import (@@ -293,6 +292,7 @@ class StealthySession(StealthySessionMixin, SyncSession):("solve_cloudflare", solve_cloudflare, self.solve_cloudflare),("selector_config", selector_config, self.selector_config),],+ CamoufoxConfig,_UNSET,)@@ -574,6 +574,7 @@ class AsyncStealthySession(StealthySessionMixin, AsyncSession):("solve_cloudflare", solve_cloudflare, self.solve_cloudflare),("selector_config", selector_config, self.selector_config),],+ CamoufoxConfig,_UNSET,)diff --git a/scrapling/engines/_browsers/_controllers.py b/scrapling/engines/_browsers/_controllers.pyindex d8f841b..bcfd0ac 100644--- a/scrapling/engines/_browsers/_controllers.py+++ b/scrapling/engines/_browsers/_controllers.py@@ -14,7 +14,7 @@ from patchright.async_api import async_playwright as async_patchrightfrom scrapling.core.utils import logfrom ._base import SyncSession, AsyncSession, DynamicSessionMixin-from ._validators import validate_fetch as _validate+from ._validators import validate_fetch as _validate, PlaywrightConfigfrom scrapling.core._types import (Any,Dict,@@ -225,6 +225,7 @@ class DynamicSession(DynamicSessionMixin, SyncSession):("load_dom", load_dom, self.load_dom),("selector_config", selector_config, self.selector_config),],+ PlaywrightConfig,_UNSET,)@@ -442,6 +443,7 @@ class AsyncDynamicSession(DynamicSessionMixin, AsyncSession):("load_dom", load_dom, self.load_dom),("selector_config", selector_config, self.selector_config),],+ PlaywrightConfig,_UNSET,)diff --git a/scrapling/engines/_browsers/_validators.py b/scrapling/engines/_browsers/_validators.pyindex 831aacb..edeaa2f 100644--- a/scrapling/engines/_browsers/_validators.py+++ b/scrapling/engines/_browsers/_validators.py@@ -1,7 +1,8 @@from pathlib import Pathfrom typing import Annotated-from dataclasses import dataclass+from functools import lru_cachefrom urllib.parse import urlparse+from dataclasses import dataclass, fieldsfrom msgspec import Struct, Meta, convert, ValidationError@@ -19,18 +20,20 @@ from scrapling.engines.toolbelt.navigation import construct_proxy_dict# Custom validators for msgspec-def _validate_file_path(value: str):+@lru_cache(8)+def _is_invalid_file_path(value: str) -> bool | str:"""Fast file path validation"""path = Path(value)if not path.exists():- raise ValueError(f"Init script path not found: {value}")+ return f"Init script path not found: {value}"if not path.is_file():- raise ValueError(f"Init script is not a file: {value}")+ return f"Init script is not a file: {value}"if not path.is_absolute():- raise ValueError(f"Init script is not a absolute path: {value}")+ return f"Init script is not a absolute path: {value}"+ return False-def _validate_addon_path(value: str):+def _validate_addon_path(value: str) -> None:"""Fast addon path validation"""path = Path(value)if not path.exists():@@ -39,22 +42,16 @@ def _validate_addon_path(value: str):raise ValueError(f"Addon path must be a directory of the extracted addon: {value}")-def _validate_cdp_url(cdp_url: str):+@lru_cache(2)+def _is_invalid_cdp_url(cdp_url: str) -> bool | str:"""Fast CDP URL validation"""- try:- # Check the scheme- if not cdp_url.startswith(("ws://", "wss://")):- raise ValueError("CDP URL must use 'ws://' or 'wss://' scheme")-- # Validate hostname and port- if not urlparse(cdp_url).netloc:- raise ValueError("Invalid hostname for the CDP URL")+ if not cdp_url.startswith(("ws://", "wss://")):+ return "CDP URL must use 'ws://' or 'wss://' scheme"- except AttributeError as e:- raise ValueError(f"Malformed CDP URL: {cdp_url}: {str(e)}")-- except Exception as e:- raise ValueError(f"Invalid CDP URL '{cdp_url}': {str(e)}")+ netloc = urlparse(cdp_url).netloc+ if not netloc:+ return "Invalid hostname for the CDP URL"+ return False# Type aliases for cleaner annotations@@ -62,7 +59,7 @@ PagesCount = Annotated[int, Meta(ge=1, le=50)]Seconds = Annotated[int, float, Meta(ge=0)]-class PlaywrightConfig(Struct, kw_only=True, frozen=False):+class PlaywrightConfig(Struct, kw_only=True, frozen=False, weakref=True):"""Configuration struct for validation"""max_pages: PagesCount = 1@@ -99,7 +96,9 @@ class PlaywrightConfig(Struct, kw_only=True, frozen=False):if self.proxy:self.proxy = construct_proxy_dict(self.proxy, as_tuple=True)if self.cdp_url:- _validate_cdp_url(self.cdp_url)+ cdp_msg = _is_invalid_cdp_url(self.cdp_url)+ if cdp_msg:+ raise ValueError(cdp_msg)if not self.cookies:self.cookies = []@@ -111,10 +110,12 @@ class PlaywrightConfig(Struct, kw_only=True, frozen=False):self.additional_args = {}if self.init_script is not None:- _validate_file_path(self.init_script)+ validation_msg = _is_invalid_file_path(self.init_script)+ if validation_msg:+ raise ValueError(validation_msg)-class CamoufoxConfig(Struct, kw_only=True, frozen=False):+class CamoufoxConfig(Struct, kw_only=True, frozen=False, weakref=True):"""Configuration struct for validation"""max_pages: PagesCount = 1@@ -152,14 +153,16 @@ class CamoufoxConfig(Struct, kw_only=True, frozen=False):if self.proxy:self.proxy = construct_proxy_dict(self.proxy, as_tuple=True)- if self.addons and isinstance(self.addons, list):+ if self.addons:for addon in self.addons:_validate_addon_path(addon)else:self.addons = []if self.init_script is not None:- _validate_file_path(self.init_script)+ validation_msg = _is_invalid_file_path(self.init_script)+ if validation_msg:+ raise ValueError(validation_msg)if not self.cookies:self.cookies = []@@ -172,27 +175,6 @@ class CamoufoxConfig(Struct, kw_only=True, frozen=False):self.additional_args = {}-# Code parts to validate `fetch` in the least possible numbers of lines overall-class FetchConfig(Struct, kw_only=True):- """Configuration struct for `fetch` calls validation"""-- google_search: bool = True- timeout: Seconds = 30000- wait: Seconds = 0- page_action: Optional[Callable] = None- extra_headers: Optional[Dict[str, str]] = None- disable_resources: bool = False- wait_selector: Optional[str] = None- wait_selector_state: SelectorWaitStates = "attached"- network_idle: bool = False- load_dom: bool = True- solve_cloudflare: bool = False- selector_config: Dict = {}-- def to_dict(self):- return {f: getattr(self, f) for f in self.__struct_fields__}--@dataclassclass _fetch_params:"""A dataclass of all parameters used by `fetch` calls"""@@ -211,7 +193,9 @@ class _fetch_params:selector_config: Dict-def validate_fetch(params: List[Tuple], sentinel=None) -> _fetch_params:+def validate_fetch(+ params: List[Tuple], model: type[PlaywrightConfig] | type[CamoufoxConfig], sentinel=None+) -> _fetch_params:result = {}overrides = {}@@ -222,32 +206,56 @@ def validate_fetch(params: List[Tuple], sentinel=None) -> _fetch_params:result[arg] = session_valueif overrides:- overrides = validate(overrides, FetchConfig).to_dict()- overrides.update(result)- return _fetch_params(**overrides)+ validated_config = validate(overrides, model)+ # Extract only the fields that _fetch_params needs from validated_config+ validated_dict = {+ f.name: getattr(validated_config, f.name)+ for f in fields(_fetch_params)+ if hasattr(validated_config, f.name)+ }+ # solve_cloudflare defaults to False for models that don't have it (PlaywrightConfig)+ validated_dict.setdefault("solve_cloudflare", False)- if not result.get("solve_cloudflare"):- result["solve_cloudflare"] = False+ validated_dict.update(result)+ return _fetch_params(**validated_dict)++ result.setdefault("solve_cloudflare", False)return _fetch_params(**result)-@overload-def validate(params: Dict, model: type[PlaywrightConfig]) -> PlaywrightConfig: ...+# Cache default values for each model to reduce validation overhead+models_default_values = {}++for _model in (CamoufoxConfig, PlaywrightConfig):+ _defaults = {}+ if hasattr(_model, "__struct_defaults__") and hasattr(_model, "__struct_fields__"):+ for field_name, default_value in zip(_model.__struct_fields__, _model.__struct_defaults__): # type: ignore+ # Skip factory defaults - these are msgspec._core.Factory instances+ if type(default_value).__name__ != "Factory":+ _defaults[field_name] = default_value++ models_default_values[_model.__name__] = _defaults.copy()+++def _filter_defaults(params: Dict, model: str) -> Dict:+ """Filter out parameters that match their default values to reduce validation overhead."""+ defaults = models_default_values[model]+ return {k: v for k, v in params.items() if k not in defaults or v != defaults[k]}@overload-def validate(params: Dict, model: type[CamoufoxConfig]) -> CamoufoxConfig: ...+def validate(params: Dict, model: type[PlaywrightConfig]) -> PlaywrightConfig: ...@overload-def validate(params: Dict, model: type[FetchConfig]) -> FetchConfig: ...+def validate(params: Dict, model: type[CamoufoxConfig]) -> CamoufoxConfig: ...-def validate(- params: Dict, model: type[PlaywrightConfig] | type[CamoufoxConfig] | type[FetchConfig]-) -> PlaywrightConfig | CamoufoxConfig | FetchConfig:+def validate(params: Dict, model: type[PlaywrightConfig] | type[CamoufoxConfig]) -> PlaywrightConfig | CamoufoxConfig:try:- return convert(params, model)+ # Filter out params with the default values (no need to validate them) to speed up validation+ filtered = _filter_defaults(params, model.__name__)+ return convert(filtered, model)except ValidationError as e:raise TypeError(f"Invalid argument type: {e}") from e