Skip to content

🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev

refactor(validation): All browser based fetchers are ~8-15% faster now in most cases

Karim shoaird4a250766685
diff --git a/scrapling/engines/_browsers/_camoufox.py b/scrapling/engines/_browsers/_camoufox.py
index ac66870..c0b2f2c 100644
--- a/scrapling/engines/_browsers/_camoufox.py
+++ b/scrapling/engines/_browsers/_camoufox.py
@@ -13,9 +13,8 @@ from playwright.async_api import (
Playwright as AsyncPlaywright,
BrowserContext as AsyncBrowserContext,
)
-from playwright._impl._errors import Error as PlaywrightError
-from ._validators import validate_fetch as _validate
+from ._validators import validate_fetch as _validate, CamoufoxConfig
from ._base import SyncSession, AsyncSession, StealthySessionMixin
from scrapling.core.utils import log
from scrapling.core._types import (
@@ -293,6 +292,7 @@ class StealthySession(StealthySessionMixin, SyncSession):
("solve_cloudflare", solve_cloudflare, self.solve_cloudflare),
("selector_config", selector_config, self.selector_config),
],
+ CamoufoxConfig,
_UNSET,
)
@@ -574,6 +574,7 @@ class AsyncStealthySession(StealthySessionMixin, AsyncSession):
("solve_cloudflare", solve_cloudflare, self.solve_cloudflare),
("selector_config", selector_config, self.selector_config),
],
+ CamoufoxConfig,
_UNSET,
)
diff --git a/scrapling/engines/_browsers/_controllers.py b/scrapling/engines/_browsers/_controllers.py
index d8f841b..bcfd0ac 100644
--- a/scrapling/engines/_browsers/_controllers.py
+++ b/scrapling/engines/_browsers/_controllers.py
@@ -14,7 +14,7 @@ from patchright.async_api import async_playwright as async_patchright
from scrapling.core.utils import log
from ._base import SyncSession, AsyncSession, DynamicSessionMixin
-from ._validators import validate_fetch as _validate
+from ._validators import validate_fetch as _validate, PlaywrightConfig
from scrapling.core._types import (
Any,
Dict,
@@ -225,6 +225,7 @@ class DynamicSession(DynamicSessionMixin, SyncSession):
("load_dom", load_dom, self.load_dom),
("selector_config", selector_config, self.selector_config),
],
+ PlaywrightConfig,
_UNSET,
)
@@ -442,6 +443,7 @@ class AsyncDynamicSession(DynamicSessionMixin, AsyncSession):
("load_dom", load_dom, self.load_dom),
("selector_config", selector_config, self.selector_config),
],
+ PlaywrightConfig,
_UNSET,
)
diff --git a/scrapling/engines/_browsers/_validators.py b/scrapling/engines/_browsers/_validators.py
index 831aacb..edeaa2f 100644
--- a/scrapling/engines/_browsers/_validators.py
+++ b/scrapling/engines/_browsers/_validators.py
@@ -1,7 +1,8 @@
from pathlib import Path
from typing import Annotated
-from dataclasses import dataclass
+from functools import lru_cache
from urllib.parse import urlparse
+from dataclasses import dataclass, fields
from msgspec import Struct, Meta, convert, ValidationError
@@ -19,18 +20,20 @@ from scrapling.engines.toolbelt.navigation import construct_proxy_dict
# Custom validators for msgspec
-def _validate_file_path(value: str):
+@lru_cache(8)
+def _is_invalid_file_path(value: str) -> bool | str:
"""Fast file path validation"""
path = Path(value)
if not path.exists():
- raise ValueError(f"Init script path not found: {value}")
+ return f"Init script path not found: {value}"
if not path.is_file():
- raise ValueError(f"Init script is not a file: {value}")
+ return f"Init script is not a file: {value}"
if not path.is_absolute():
- raise ValueError(f"Init script is not a absolute path: {value}")
+ return f"Init script is not a absolute path: {value}"
+ return False
-def _validate_addon_path(value: str):
+def _validate_addon_path(value: str) -> None:
"""Fast addon path validation"""
path = Path(value)
if not path.exists():
@@ -39,22 +42,16 @@ def _validate_addon_path(value: str):
raise ValueError(f"Addon path must be a directory of the extracted addon: {value}")
-def _validate_cdp_url(cdp_url: str):
+@lru_cache(2)
+def _is_invalid_cdp_url(cdp_url: str) -> bool | str:
"""Fast CDP URL validation"""
- try:
- # Check the scheme
- if not cdp_url.startswith(("ws://", "wss://")):
- raise ValueError("CDP URL must use 'ws://' or 'wss://' scheme")
-
- # Validate hostname and port
- if not urlparse(cdp_url).netloc:
- raise ValueError("Invalid hostname for the CDP URL")
+ if not cdp_url.startswith(("ws://", "wss://")):
+ return "CDP URL must use 'ws://' or 'wss://' scheme"
- except AttributeError as e:
- raise ValueError(f"Malformed CDP URL: {cdp_url}: {str(e)}")
-
- except Exception as e:
- raise ValueError(f"Invalid CDP URL '{cdp_url}': {str(e)}")
+ netloc = urlparse(cdp_url).netloc
+ if not netloc:
+ return "Invalid hostname for the CDP URL"
+ return False
# Type aliases for cleaner annotations
@@ -62,7 +59,7 @@ PagesCount = Annotated[int, Meta(ge=1, le=50)]
Seconds = Annotated[int, float, Meta(ge=0)]
-class PlaywrightConfig(Struct, kw_only=True, frozen=False):
+class PlaywrightConfig(Struct, kw_only=True, frozen=False, weakref=True):
"""Configuration struct for validation"""
max_pages: PagesCount = 1
@@ -99,7 +96,9 @@ class PlaywrightConfig(Struct, kw_only=True, frozen=False):
if self.proxy:
self.proxy = construct_proxy_dict(self.proxy, as_tuple=True)
if self.cdp_url:
- _validate_cdp_url(self.cdp_url)
+ cdp_msg = _is_invalid_cdp_url(self.cdp_url)
+ if cdp_msg:
+ raise ValueError(cdp_msg)
if not self.cookies:
self.cookies = []
@@ -111,10 +110,12 @@ class PlaywrightConfig(Struct, kw_only=True, frozen=False):
self.additional_args = {}
if self.init_script is not None:
- _validate_file_path(self.init_script)
+ validation_msg = _is_invalid_file_path(self.init_script)
+ if validation_msg:
+ raise ValueError(validation_msg)
-class CamoufoxConfig(Struct, kw_only=True, frozen=False):
+class CamoufoxConfig(Struct, kw_only=True, frozen=False, weakref=True):
"""Configuration struct for validation"""
max_pages: PagesCount = 1
@@ -152,14 +153,16 @@ class CamoufoxConfig(Struct, kw_only=True, frozen=False):
if self.proxy:
self.proxy = construct_proxy_dict(self.proxy, as_tuple=True)
- if self.addons and isinstance(self.addons, list):
+ if self.addons:
for addon in self.addons:
_validate_addon_path(addon)
else:
self.addons = []
if self.init_script is not None:
- _validate_file_path(self.init_script)
+ validation_msg = _is_invalid_file_path(self.init_script)
+ if validation_msg:
+ raise ValueError(validation_msg)
if not self.cookies:
self.cookies = []
@@ -172,27 +175,6 @@ class CamoufoxConfig(Struct, kw_only=True, frozen=False):
self.additional_args = {}
-# Code parts to validate `fetch` in the least possible numbers of lines overall
-class FetchConfig(Struct, kw_only=True):
- """Configuration struct for `fetch` calls validation"""
-
- google_search: bool = True
- timeout: Seconds = 30000
- wait: Seconds = 0
- page_action: Optional[Callable] = None
- extra_headers: Optional[Dict[str, str]] = None
- disable_resources: bool = False
- wait_selector: Optional[str] = None
- wait_selector_state: SelectorWaitStates = "attached"
- network_idle: bool = False
- load_dom: bool = True
- solve_cloudflare: bool = False
- selector_config: Dict = {}
-
- def to_dict(self):
- return {f: getattr(self, f) for f in self.__struct_fields__}
-
-
@dataclass
class _fetch_params:
"""A dataclass of all parameters used by `fetch` calls"""
@@ -211,7 +193,9 @@ class _fetch_params:
selector_config: Dict
-def validate_fetch(params: List[Tuple], sentinel=None) -> _fetch_params:
+def validate_fetch(
+ params: List[Tuple], model: type[PlaywrightConfig] | type[CamoufoxConfig], sentinel=None
+) -> _fetch_params:
result = {}
overrides = {}
@@ -222,32 +206,56 @@ def validate_fetch(params: List[Tuple], sentinel=None) -> _fetch_params:
result[arg] = session_value
if overrides:
- overrides = validate(overrides, FetchConfig).to_dict()
- overrides.update(result)
- return _fetch_params(**overrides)
+ validated_config = validate(overrides, model)
+ # Extract only the fields that _fetch_params needs from validated_config
+ validated_dict = {
+ f.name: getattr(validated_config, f.name)
+ for f in fields(_fetch_params)
+ if hasattr(validated_config, f.name)
+ }
+ # solve_cloudflare defaults to False for models that don't have it (PlaywrightConfig)
+ validated_dict.setdefault("solve_cloudflare", False)
- if not result.get("solve_cloudflare"):
- result["solve_cloudflare"] = False
+ validated_dict.update(result)
+ return _fetch_params(**validated_dict)
+
+ result.setdefault("solve_cloudflare", False)
return _fetch_params(**result)
-@overload
-def validate(params: Dict, model: type[PlaywrightConfig]) -> PlaywrightConfig: ...
+# Cache default values for each model to reduce validation overhead
+models_default_values = {}
+
+for _model in (CamoufoxConfig, PlaywrightConfig):
+ _defaults = {}
+ if hasattr(_model, "__struct_defaults__") and hasattr(_model, "__struct_fields__"):
+ for field_name, default_value in zip(_model.__struct_fields__, _model.__struct_defaults__): # type: ignore
+ # Skip factory defaults - these are msgspec._core.Factory instances
+ if type(default_value).__name__ != "Factory":
+ _defaults[field_name] = default_value
+
+ models_default_values[_model.__name__] = _defaults.copy()
+
+
+def _filter_defaults(params: Dict, model: str) -> Dict:
+ """Filter out parameters that match their default values to reduce validation overhead."""
+ defaults = models_default_values[model]
+ return {k: v for k, v in params.items() if k not in defaults or v != defaults[k]}
@overload
-def validate(params: Dict, model: type[CamoufoxConfig]) -> CamoufoxConfig: ...
+def validate(params: Dict, model: type[PlaywrightConfig]) -> PlaywrightConfig: ...
@overload
-def validate(params: Dict, model: type[FetchConfig]) -> FetchConfig: ...
+def validate(params: Dict, model: type[CamoufoxConfig]) -> CamoufoxConfig: ...
-def validate(
- params: Dict, model: type[PlaywrightConfig] | type[CamoufoxConfig] | type[FetchConfig]
-) -> PlaywrightConfig | CamoufoxConfig | FetchConfig:
+def validate(params: Dict, model: type[PlaywrightConfig] | type[CamoufoxConfig]) -> PlaywrightConfig | CamoufoxConfig:
try:
- return convert(params, model)
+ # Filter out params with the default values (no need to validate them) to speed up validation
+ filtered = _filter_defaults(params, model.__name__)
+ return convert(filtered, model)
except ValidationError as e:
raise TypeError(f"Invalid argument type: {e}") from e