Skip to content

🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev

perf: Improving all browser-based fetches by ~15%

Karim shoair82b09620b28f
diff --git a/scrapling/engines/_browsers/_base.py b/scrapling/engines/_browsers/_base.py
index 703a915..69681ee 100644
--- a/scrapling/engines/_browsers/_base.py
+++ b/scrapling/engines/_browsers/_base.py
@@ -60,11 +60,6 @@ class SyncSession:
return self.page_pool.add_page(page)
- @staticmethod
- def _get_with_precedence(request_value: Any, session_value: Any, sentinel_value: object) -> Any:
- """Get value with request-level priority over session-level"""
- return request_value if request_value is not sentinel_value else session_value
-
def get_pool_stats(self) -> Dict[str, int]:
"""Get statistics about the current page pool"""
return {
diff --git a/scrapling/engines/_browsers/_camoufox.py b/scrapling/engines/_browsers/_camoufox.py
index 18ec633..5f0e795 100644
--- a/scrapling/engines/_browsers/_camoufox.py
+++ b/scrapling/engines/_browsers/_camoufox.py
@@ -16,7 +16,7 @@ from playwright.async_api import (
)
from playwright._impl._errors import Error as PlaywrightError
-from ._validators import validate, CamoufoxConfig
+from ._validators import validate_fetch as _validate
from ._base import SyncSession, AsyncSession, StealthySessionMixin
from scrapling.core.utils import log
from scrapling.core._types import (
@@ -297,23 +297,22 @@ class StealthySession(StealthySessionMixin, SyncSession):
:param selector_config: The arguments that will be passed in the end while creating the final Selector's class.
:return: A `Response` object.
"""
- # Validate all resolved parameters
- params = validate(
- dict(
- google_search=self._get_with_precedence(google_search, self.google_search, _UNSET),
- timeout=self._get_with_precedence(timeout, self.timeout, _UNSET),
- wait=self._get_with_precedence(wait, self.wait, _UNSET),
- page_action=self._get_with_precedence(page_action, self.page_action, _UNSET),
- extra_headers=self._get_with_precedence(extra_headers, self.extra_headers, _UNSET),
- disable_resources=self._get_with_precedence(disable_resources, self.disable_resources, _UNSET),
- wait_selector=self._get_with_precedence(wait_selector, self.wait_selector, _UNSET),
- wait_selector_state=self._get_with_precedence(wait_selector_state, self.wait_selector_state, _UNSET),
- network_idle=self._get_with_precedence(network_idle, self.network_idle, _UNSET),
- load_dom=self._get_with_precedence(load_dom, self.load_dom, _UNSET),
- solve_cloudflare=self._get_with_precedence(solve_cloudflare, self.solve_cloudflare, _UNSET),
- selector_config=self._get_with_precedence(selector_config, self.selector_config, _UNSET),
- ),
- CamoufoxConfig,
+ params = _validate(
+ [
+ ("google_search", google_search, self.google_search),
+ ("timeout", timeout, self.timeout),
+ ("wait", wait, self.wait),
+ ("page_action", page_action, self.page_action),
+ ("extra_headers", extra_headers, self.extra_headers),
+ ("disable_resources", disable_resources, self.disable_resources),
+ ("wait_selector", wait_selector, self.wait_selector),
+ ("wait_selector_state", wait_selector_state, self.wait_selector_state),
+ ("network_idle", network_idle, self.network_idle),
+ ("load_dom", load_dom, self.load_dom),
+ ("solve_cloudflare", solve_cloudflare, self.solve_cloudflare),
+ ("selector_config", selector_config, self.selector_config),
+ ],
+ _UNSET,
)
if self._closed: # pragma: no cover
@@ -617,22 +616,22 @@ class AsyncStealthySession(StealthySessionMixin, AsyncSession):
:param selector_config: The arguments that will be passed in the end while creating the final Selector's class.
:return: A `Response` object.
"""
- params = validate(
- dict(
- google_search=self._get_with_precedence(google_search, self.google_search, _UNSET),
- timeout=self._get_with_precedence(timeout, self.timeout, _UNSET),
- wait=self._get_with_precedence(wait, self.wait, _UNSET),
- page_action=self._get_with_precedence(page_action, self.page_action, _UNSET),
- extra_headers=self._get_with_precedence(extra_headers, self.extra_headers, _UNSET),
- disable_resources=self._get_with_precedence(disable_resources, self.disable_resources, _UNSET),
- wait_selector=self._get_with_precedence(wait_selector, self.wait_selector, _UNSET),
- wait_selector_state=self._get_with_precedence(wait_selector_state, self.wait_selector_state, _UNSET),
- network_idle=self._get_with_precedence(network_idle, self.network_idle, _UNSET),
- load_dom=self._get_with_precedence(load_dom, self.load_dom, _UNSET),
- solve_cloudflare=self._get_with_precedence(solve_cloudflare, self.solve_cloudflare, _UNSET),
- selector_config=self._get_with_precedence(selector_config, self.selector_config, _UNSET),
- ),
- CamoufoxConfig,
+ params = _validate(
+ [
+ ("google_search", google_search, self.google_search),
+ ("timeout", timeout, self.timeout),
+ ("wait", wait, self.wait),
+ ("page_action", page_action, self.page_action),
+ ("extra_headers", extra_headers, self.extra_headers),
+ ("disable_resources", disable_resources, self.disable_resources),
+ ("wait_selector", wait_selector, self.wait_selector),
+ ("wait_selector_state", wait_selector_state, self.wait_selector_state),
+ ("network_idle", network_idle, self.network_idle),
+ ("load_dom", load_dom, self.load_dom),
+ ("solve_cloudflare", solve_cloudflare, self.solve_cloudflare),
+ ("selector_config", selector_config, self.selector_config),
+ ],
+ _UNSET,
)
if self._closed: # pragma: no cover
diff --git a/scrapling/engines/_browsers/_controllers.py b/scrapling/engines/_browsers/_controllers.py
index 19362e7..b4b6276 100644
--- a/scrapling/engines/_browsers/_controllers.py
+++ b/scrapling/engines/_browsers/_controllers.py
@@ -16,7 +16,7 @@ from patchright.async_api import async_playwright as async_patchright
from scrapling.core.utils import log
from ._base import SyncSession, AsyncSession, DynamicSessionMixin
-from ._validators import validate, PlaywrightConfig
+from ._validators import validate_fetch as _validate
from scrapling.core._types import (
Dict,
List,
@@ -224,22 +224,21 @@ class DynamicSession(DynamicSessionMixin, SyncSession):
:param selector_config: The arguments that will be passed in the end while creating the final Selector's class.
:return: A `Response` object.
"""
- # Validate all resolved parameters
- params = validate(
- dict(
- google_search=self._get_with_precedence(google_search, self.google_search, _UNSET),
- timeout=self._get_with_precedence(timeout, self.timeout, _UNSET),
- wait=self._get_with_precedence(wait, self.wait, _UNSET),
- page_action=self._get_with_precedence(page_action, self.page_action, _UNSET),
- extra_headers=self._get_with_precedence(extra_headers, self.extra_headers, _UNSET),
- disable_resources=self._get_with_precedence(disable_resources, self.disable_resources, _UNSET),
- wait_selector=self._get_with_precedence(wait_selector, self.wait_selector, _UNSET),
- wait_selector_state=self._get_with_precedence(wait_selector_state, self.wait_selector_state, _UNSET),
- network_idle=self._get_with_precedence(network_idle, self.network_idle, _UNSET),
- load_dom=self._get_with_precedence(load_dom, self.load_dom, _UNSET),
- selector_config=self._get_with_precedence(selector_config, self.selector_config, _UNSET),
- ),
- PlaywrightConfig,
+ params = _validate(
+ [
+ ("google_search", google_search, self.google_search),
+ ("timeout", timeout, self.timeout),
+ ("wait", wait, self.wait),
+ ("page_action", page_action, self.page_action),
+ ("extra_headers", extra_headers, self.extra_headers),
+ ("disable_resources", disable_resources, self.disable_resources),
+ ("wait_selector", wait_selector, self.wait_selector),
+ ("wait_selector_state", wait_selector_state, self.wait_selector_state),
+ ("network_idle", network_idle, self.network_idle),
+ ("load_dom", load_dom, self.load_dom),
+ ("selector_config", selector_config, self.selector_config),
+ ],
+ _UNSET,
)
if self._closed: # pragma: no cover
@@ -471,22 +470,21 @@ class AsyncDynamicSession(DynamicSessionMixin, AsyncSession):
:param selector_config: The arguments that will be passed in the end while creating the final Selector's class.
:return: A `Response` object.
"""
- # Validate all resolved parameters
- params = validate(
- dict(
- google_search=self._get_with_precedence(google_search, self.google_search, _UNSET),
- timeout=self._get_with_precedence(timeout, self.timeout, _UNSET),
- wait=self._get_with_precedence(wait, self.wait, _UNSET),
- page_action=self._get_with_precedence(page_action, self.page_action, _UNSET),
- extra_headers=self._get_with_precedence(extra_headers, self.extra_headers, _UNSET),
- disable_resources=self._get_with_precedence(disable_resources, self.disable_resources, _UNSET),
- wait_selector=self._get_with_precedence(wait_selector, self.wait_selector, _UNSET),
- wait_selector_state=self._get_with_precedence(wait_selector_state, self.wait_selector_state, _UNSET),
- network_idle=self._get_with_precedence(network_idle, self.network_idle, _UNSET),
- load_dom=self._get_with_precedence(load_dom, self.load_dom, _UNSET),
- selector_config=self._get_with_precedence(selector_config, self.selector_config, _UNSET),
- ),
- PlaywrightConfig,
+ params = _validate(
+ [
+ ("google_search", google_search, self.google_search),
+ ("timeout", timeout, self.timeout),
+ ("wait", wait, self.wait),
+ ("page_action", page_action, self.page_action),
+ ("extra_headers", extra_headers, self.extra_headers),
+ ("disable_resources", disable_resources, self.disable_resources),
+ ("wait_selector", wait_selector, self.wait_selector),
+ ("wait_selector_state", wait_selector_state, self.wait_selector_state),
+ ("network_idle", network_idle, self.network_idle),
+ ("load_dom", load_dom, self.load_dom),
+ ("selector_config", selector_config, self.selector_config),
+ ],
+ _UNSET,
)
if self._closed: # pragma: no cover
diff --git a/scrapling/engines/_browsers/_validators.py b/scrapling/engines/_browsers/_validators.py
index ca64942..a2d7c60 100644
--- a/scrapling/engines/_browsers/_validators.py
+++ b/scrapling/engines/_browsers/_validators.py
@@ -1,21 +1,69 @@
-from msgspec import Struct, convert, ValidationError
-from urllib.parse import urlparse
from pathlib import Path
+from typing import Annotated
+from dataclasses import dataclass
+from urllib.parse import urlparse
+
+from msgspec import Struct, Meta, convert, ValidationError
from scrapling.core._types import (
- Optional,
Dict,
- Callable,
List,
+ Tuple,
+ Optional,
+ Callable,
SelectorWaitStates,
)
from scrapling.engines.toolbelt.navigation import construct_proxy_dict
+# Custom validators for msgspec
+def _validate_file_path(value: str):
+ """Fast file path validation"""
+ path = Path(value)
+ if not path.exists():
+ raise ValueError(f"Init script path not found: {value}")
+ if not path.is_file():
+ raise ValueError(f"Init script is not a file: {value}")
+ if not path.is_absolute():
+ raise ValueError(f"Init script is not a absolute path: {value}")
+
+
+def _validate_addon_path(value: str):
+ """Fast addon path validation"""
+ path = Path(value)
+ if not path.exists():
+ raise FileNotFoundError(f"Addon path not found: {value}")
+ if not path.is_dir():
+ raise ValueError(f"Addon path must be a directory of the extracted addon: {value}")
+
+
+def _validate_cdp_url(cdp_url: str):
+ """Fast CDP URL validation"""
+ try:
+ # Check the scheme
+ if not cdp_url.startswith(("ws://", "wss://")):
+ raise ValueError("CDP URL must use 'ws://' or 'wss://' scheme")
+
+ # Validate hostname and port
+ if not urlparse(cdp_url).netloc:
+ raise ValueError("Invalid hostname for the CDP URL")
+
+ except AttributeError as e:
+ raise ValueError(f"Malformed CDP URL: {cdp_url}: {str(e)}")
+
+ except Exception as e:
+ raise ValueError(f"Invalid CDP URL '{cdp_url}': {str(e)}")
+
+
+# Type aliases for cleaner annotations
+PagesCount = Annotated[int, Meta(ge=1, le=50)]
+Seconds = Annotated[int, float, Meta(ge=0)]
+
+
class PlaywrightConfig(Struct, kw_only=True, frozen=False):
"""Configuration struct for validation"""
- max_pages: int = 1
+ max_pages: PagesCount = 1
cdp_url: Optional[str] = None
headless: bool = True
google_search: bool = True
@@ -23,13 +71,13 @@ class PlaywrightConfig(Struct, kw_only=True, frozen=False):
disable_webgl: bool = False
real_chrome: bool = False
stealth: bool = False
- wait: int | float = 0
+ wait: Seconds = 0
page_action: Optional[Callable] = None
proxy: Optional[str | Dict[str, str]] = None # The default value for proxy in Playwright's source is `None`
locale: str = "en-US"
extra_headers: Optional[Dict[str, str]] = None
useragent: Optional[str] = None
- timeout: int | float = 30000
+ timeout: Seconds = 30000
init_script: Optional[str] = None
disable_resources: bool = False
wait_selector: Optional[str] = None
@@ -41,52 +89,26 @@ class PlaywrightConfig(Struct, kw_only=True, frozen=False):
def __post_init__(self):
"""Custom validation after msgspec validation"""
- if self.max_pages < 1 or self.max_pages > 50:
- raise ValueError("max_pages must be between 1 and 50")
- if self.timeout < 0:
- raise ValueError("timeout must be >= 0")
if self.page_action and not callable(self.page_action):
raise TypeError(f"page_action must be callable, got {type(self.page_action).__name__}")
if self.proxy:
self.proxy = construct_proxy_dict(self.proxy, as_tuple=True)
if self.cdp_url:
- self.__validate_cdp(self.cdp_url)
+ _validate_cdp_url(self.cdp_url)
+
if not self.cookies:
self.cookies = []
if not self.selector_config:
self.selector_config = {}
if self.init_script is not None:
- script_path = Path(self.init_script)
- if not script_path.exists():
- raise ValueError("Init script path not found")
- elif not script_path.is_file():
- raise ValueError("Init script is not a file")
- elif not script_path.is_absolute():
- raise ValueError("Init script is not a absolute path")
-
- @staticmethod
- def __validate_cdp(cdp_url):
- try:
- # Check the scheme
- if not cdp_url.startswith(("ws://", "wss://")):
- raise ValueError("CDP URL must use 'ws://' or 'wss://' scheme")
-
- # Validate hostname and port
- if not urlparse(cdp_url).netloc:
- raise ValueError("Invalid hostname for the CDP URL")
-
- except AttributeError as e:
- raise ValueError(f"Malformed CDP URL: {cdp_url}: {str(e)}")
-
- except Exception as e:
- raise ValueError(f"Invalid CDP URL '{cdp_url}': {str(e)}")
+ _validate_file_path(self.init_script)
class CamoufoxConfig(Struct, kw_only=True, frozen=False):
"""Configuration struct for validation"""
- max_pages: int = 1
+ max_pages: PagesCount = 1
headless: bool = True # noqa: F821
block_images: bool = False
disable_resources: bool = False
@@ -96,8 +118,8 @@ class CamoufoxConfig(Struct, kw_only=True, frozen=False):
load_dom: bool = True
humanize: bool | float = True
solve_cloudflare: bool = False
- wait: int | float = 0
- timeout: int | float = 30000
+ wait: Seconds = 0
+ timeout: Seconds = 30000
init_script: Optional[str] = None
page_action: Optional[Callable] = None
wait_selector: Optional[str] = None
@@ -115,38 +137,23 @@ class CamoufoxConfig(Struct, kw_only=True, frozen=False):
def __post_init__(self):
"""Custom validation after msgspec validation"""
- if self.max_pages < 1 or self.max_pages > 50:
- raise ValueError("max_pages must be between 1 and 50")
- if self.timeout < 0:
- raise ValueError("timeout must be >= 0")
if self.page_action and not callable(self.page_action):
raise TypeError(f"page_action must be callable, got {type(self.page_action).__name__}")
if self.proxy:
self.proxy = construct_proxy_dict(self.proxy, as_tuple=True)
- if not self.addons:
- self.addons = []
- else:
+ if self.addons and isinstance(self.addons, list):
for addon in self.addons:
- addon_path = Path(addon)
- if not addon_path.exists():
- raise FileNotFoundError(f"Addon's path not found: {addon}")
- elif not addon_path.is_dir():
- raise ValueError(
- f"Addon's path is not a folder, you need to pass a folder of the extracted addon: {addon}"
- )
+ _validate_addon_path(addon)
+ else:
+ self.addons = []
if self.init_script is not None:
- script_path = Path(self.init_script)
- if not script_path.exists():
- raise ValueError("Init script path not found")
- elif not script_path.is_file():
- raise ValueError("Init script is not a file")
- elif not script_path.is_absolute():
- raise ValueError("Init script is not a absolute path")
+ _validate_file_path(self.init_script)
if not self.cookies:
self.cookies = []
+ # Cloudflare timeout adjustment
if self.solve_cloudflare and self.timeout < 60_000:
self.timeout = 60_000
if not self.selector_config:
@@ -155,10 +162,68 @@ class CamoufoxConfig(Struct, kw_only=True, frozen=False):
self.additional_args = {}
-def validate(params, model):
+# Code parts to validate `fetch` in the least possible numbers of lines overall
+class FetchConfig(Struct, kw_only=True):
+ """Configuration struct for `fetch` calls validation"""
+
+ google_search: bool = True
+ timeout: Seconds = 30000
+ wait: Seconds = 0
+ page_action: Optional[Callable] = None
+ extra_headers: Optional[Dict[str, str]] = None
+ disable_resources: bool = False
+ wait_selector: Optional[str] = None
+ wait_selector_state: SelectorWaitStates = "attached"
+ network_idle: bool = False
+ load_dom: bool = True
+ solve_cloudflare: bool = False
+ selector_config: Optional[Dict] = {}
+
+ def to_dict(self):
+ return {f: getattr(self, f) for f in self.__struct_fields__}
+
+
+@dataclass
+class _fetch_params:
+ """A dataclass of all parameters used by `fetch` calls"""
+
+ google_search: bool
+ timeout: Seconds
+ wait: Seconds
+ page_action: Optional[Callable]
+ extra_headers: Optional[Dict[str, str]]
+ disable_resources: bool
+ wait_selector: Optional[str]
+ wait_selector_state: SelectorWaitStates
+ network_idle: bool
+ load_dom: bool
+ solve_cloudflare: bool
+ selector_config: Optional[Dict]
+
+
+def validate_fetch(params: List[Tuple], sentinel=None) -> _fetch_params:
+ result = {}
+ overrides = {}
+
+ for arg, request_value, session_value in params:
+ if request_value is not sentinel:
+ overrides[arg] = request_value
+ else:
+ result[arg] = session_value
+
+ if overrides:
+ overrides = validate(overrides, FetchConfig).to_dict()
+ overrides.update(result)
+ return _fetch_params(**overrides)
+
+ if not result.get("solve_cloudflare"):
+ result["solve_cloudflare"] = False
+
+ return _fetch_params(**result)
+
+
+def validate(params: Dict, model) -> PlaywrightConfig | CamoufoxConfig | FetchConfig:
try:
- config = convert(params, model)
+ return convert(params, model)
except ValidationError as e:
- raise TypeError(f"Invalid argument type: {e}")
-
- return config
+ raise TypeError(f"Invalid argument type: {e}") from e