🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev
Python aiai-scrapingautomationcrawlercrawlingcrawling-pythondatadata-extractionmcpmcp-serverplaywrightpythonscrapingselectorsstealthweb-scraperweb-scrapingweb-scraping-pythonwebscrapingxpath
feat(fetcher): Make `impersonate` able to randomize fingerprint
Karim shoair3565f9d5009b
diff --git a/scrapling/cli.py b/scrapling/cli.pyindex 25fdce3..22f2dd1 100644--- a/scrapling/cli.py+++ b/scrapling/cli.py@@ -2,6 +2,8 @@ from pathlib import Pathfrom subprocess import check_outputfrom sys import executable as python_executable+from curl_cffi.requests import impersonate+from scrapling.core.utils import logfrom scrapling.engines.toolbelt.custom import Responsefrom scrapling.core.utils._shell import _CookieParser, _ParseHeaders@@ -84,7 +86,7 @@ def __BuildRequest(headers: List[str], cookies: str, params: str, json: Optional# Parse parametersparsed_headers, parsed_cookies, parsed_params, parsed_json = __ParseExtractArguments(headers, cookies, params, json)# Build request arguments- request_kwargs = {+ request_kwargs: Dict[str, Any] = {"headers": parsed_headers if parsed_headers else None,"cookies": parsed_cookies if parsed_cookies else None,}@@ -95,6 +97,10 @@ def __BuildRequest(headers: List[str], cookies: str, params: str, json: Optionalif "proxy" in kwargs:request_kwargs["proxy"] = kwargs.pop("proxy")+ # Parse impersonate parameter if it contains commas (for random selection)+ if "impersonate" in kwargs and "," in (kwargs.get("impersonate") or ""):+ kwargs["impersonate"] = [browser.strip() for browser in kwargs["impersonate"].split(",")]+return {**request_kwargs, **kwargs}@@ -225,7 +231,10 @@ def extract():default=True,help="Whether to verify SSL certificates (default: True)",)-@option("--impersonate", help="Browser to impersonate (e.g., chrome, firefox).")+@option(+ "--impersonate",+ help="Browser to impersonate. Can be a single browser (e.g., chrome) or comma-separated list for random selection (e.g., chrome,firefox,safari).",+)@option("--stealthy-headers/--no-stealthy-headers",default=True,@@ -318,7 +327,10 @@ def get(default=True,help="Whether to verify SSL certificates (default: True)",)-@option("--impersonate", help="Browser to impersonate (e.g., chrome, firefox).")+@option(+ "--impersonate",+ help="Browser to impersonate. Can be a single browser (e.g., chrome) or comma-separated list for random selection (e.g., chrome,firefox,safari).",+)@option("--stealthy-headers/--no-stealthy-headers",default=True,@@ -412,7 +424,10 @@ def post(default=True,help="Whether to verify SSL certificates (default: True)",)-@option("--impersonate", help="Browser to impersonate (e.g., chrome, firefox).")+@option(+ "--impersonate",+ help="Browser to impersonate. Can be a single browser (e.g., chrome) or comma-separated list for random selection (e.g., chrome,firefox,safari).",+)@option("--stealthy-headers/--no-stealthy-headers",default=True,@@ -504,7 +519,10 @@ def put(default=True,help="Whether to verify SSL certificates (default: True)",)-@option("--impersonate", help="Browser to impersonate (e.g., chrome, firefox).")+@option(+ "--impersonate",+ help="Browser to impersonate. Can be a single browser (e.g., chrome) or comma-separated list for random selection (e.g., chrome,firefox,safari).",+)@option("--stealthy-headers/--no-stealthy-headers",default=True,diff --git a/scrapling/core/ai.py b/scrapling/core/ai.pyindex d497e57..4d5277a 100644--- a/scrapling/core/ai.py+++ b/scrapling/core/ai.py@@ -5,6 +5,7 @@ from pydantic import BaseModel, Fieldfrom scrapling.core.shell import Convertorfrom scrapling.engines.toolbelt.custom import Response as _ScraplingResponse+from scrapling.engines.static import ImpersonateTypefrom scrapling.fetchers import (Fetcher,FetcherSession,@@ -24,9 +25,6 @@ from scrapling.core._types import (SelectorWaitStates,Generator,)-from curl_cffi.requests import (- BrowserTypeLiteral,-)class ResponseModel(BaseModel):@@ -46,7 +44,7 @@ class ScraplingMCPServer:@staticmethoddef get(url: str,- impersonate: Optional[BrowserTypeLiteral] = "chrome",+ impersonate: ImpersonateType = "chrome",extraction_type: extraction_types = "markdown",css_selector: Optional[str] = None,main_content_only: bool = True,@@ -124,7 +122,7 @@ class ScraplingMCPServer:@staticmethodasync def bulk_get(urls: Tuple[str, ...],- impersonate: Optional[BrowserTypeLiteral] = "chrome",+ impersonate: ImpersonateType = "chrome",extraction_type: extraction_types = "markdown",css_selector: Optional[str] = None,main_content_only: bool = True,diff --git a/scrapling/engines/static.py b/scrapling/engines/static.pyindex dd54168..6f68737 100644--- a/scrapling/engines/static.py+++ b/scrapling/engines/static.py@@ -1,4 +1,5 @@from abc import ABC+from random import choicefrom time import sleep as time_sleepfrom asyncio import sleep as asyncio_sleep@@ -31,12 +32,29 @@ from .toolbelt.fingerprints import generate_convincing_referer, generate_headers_UNSET: Any = object()_NO_SESSION: Any = object()+# Type alias for `impersonate` parameter - accepts a single browser or list of browsers+ImpersonateType = BrowserTypeLiteral | List[BrowserTypeLiteral] | None+++def _select_random_browser(impersonate: ImpersonateType) -> Optional[BrowserTypeLiteral]:+ """+ Handle browser selection logic for the ` impersonate ` parameter.++ If impersonate is a list, randomly select one browser from it.+ If it's a string or None, return as is.+ """+ if isinstance(impersonate, list):+ if not impersonate:+ return None+ return choice(impersonate)+ return impersonate+class _ConfigurationLogic(ABC):# Core Logic Handler (Internal Engine)def __init__(self,- impersonate: Optional[BrowserTypeLiteral] = "chrome",+ impersonate: ImpersonateType = "chrome",http3: Optional[bool] = False,stealthy_headers: Optional[bool] = True,proxies: Optional[Dict[str, str]] = None,@@ -76,7 +94,9 @@ class _ConfigurationLogic(ABC):def _merge_request_args(self, **method_kwargs) -> Dict[str, Any]:"""Merge request-specific arguments with default session arguments."""url = method_kwargs.pop("url")- impersonate = self._get_with_precedence(method_kwargs.pop("impersonate"), self._default_impersonate)+ impersonate = _select_random_browser(+ self._get_with_precedence(method_kwargs.pop("impersonate"), self._default_impersonate)+ )http3_enabled = self._get_with_precedence(method_kwargs.pop("http3"), self._default_http3)final_args = {"url": url,@@ -146,7 +166,7 @@ class _ConfigurationLogic(ABC):class _SyncSessionLogic(_ConfigurationLogic):def __init__(self,- impersonate: Optional[BrowserTypeLiteral] = "chrome",+ impersonate: ImpersonateType = "chrome",http3: Optional[bool] = False,stealthy_headers: Optional[bool] = True,proxies: Optional[Dict[str, str]] = None,@@ -261,7 +281,7 @@ class _SyncSessionLogic(_ConfigurationLogic):auth: Optional[Tuple[str, str]] = None,verify: Optional[bool] = _UNSET,cert: Optional[str | Tuple[str, str]] = _UNSET,- impersonate: Optional[BrowserTypeLiteral] = _UNSET,+ impersonate: ImpersonateType = _UNSET,http3: Optional[bool] = _UNSET,stealthy_headers: Optional[bool] = _UNSET,**kwargs,@@ -334,7 +354,7 @@ class _SyncSessionLogic(_ConfigurationLogic):auth: Optional[Tuple[str, str]] = None,verify: Optional[bool] = _UNSET,cert: Optional[str | Tuple[str, str]] = _UNSET,- impersonate: Optional[BrowserTypeLiteral] = _UNSET,+ impersonate: ImpersonateType = _UNSET,http3: Optional[bool] = _UNSET,stealthy_headers: Optional[bool] = _UNSET,**kwargs,@@ -411,7 +431,7 @@ class _SyncSessionLogic(_ConfigurationLogic):auth: Optional[Tuple[str, str]] = None,verify: Optional[bool] = _UNSET,cert: Optional[str | Tuple[str, str]] = _UNSET,- impersonate: Optional[BrowserTypeLiteral] = _UNSET,+ impersonate: ImpersonateType = _UNSET,http3: Optional[bool] = _UNSET,stealthy_headers: Optional[bool] = _UNSET,**kwargs,@@ -488,7 +508,7 @@ class _SyncSessionLogic(_ConfigurationLogic):auth: Optional[Tuple[str, str]] = None,verify: Optional[bool] = _UNSET,cert: Optional[str | Tuple[str, str]] = _UNSET,- impersonate: Optional[BrowserTypeLiteral] = _UNSET,+ impersonate: ImpersonateType = _UNSET,http3: Optional[bool] = _UNSET,stealthy_headers: Optional[bool] = _UNSET,**kwargs,@@ -552,7 +572,7 @@ class _SyncSessionLogic(_ConfigurationLogic):class _ASyncSessionLogic(_ConfigurationLogic):def __init__(self,- impersonate: Optional[BrowserTypeLiteral] = "chrome",+ impersonate: ImpersonateType = "chrome",http3: Optional[bool] = False,stealthy_headers: Optional[bool] = True,proxies: Optional[Dict[str, str]] = None,@@ -669,7 +689,7 @@ class _ASyncSessionLogic(_ConfigurationLogic):auth: Optional[Tuple[str, str]] = None,verify: Optional[bool] = _UNSET,cert: Optional[str | Tuple[str, str]] = _UNSET,- impersonate: Optional[BrowserTypeLiteral] = _UNSET,+ impersonate: ImpersonateType = _UNSET,http3: Optional[bool] = _UNSET,stealthy_headers: Optional[bool] = _UNSET,**kwargs,@@ -742,7 +762,7 @@ class _ASyncSessionLogic(_ConfigurationLogic):auth: Optional[Tuple[str, str]] = None,verify: Optional[bool] = _UNSET,cert: Optional[str | Tuple[str, str]] = _UNSET,- impersonate: Optional[BrowserTypeLiteral] = _UNSET,+ impersonate: ImpersonateType = _UNSET,http3: Optional[bool] = _UNSET,stealthy_headers: Optional[bool] = _UNSET,**kwargs,@@ -819,7 +839,7 @@ class _ASyncSessionLogic(_ConfigurationLogic):auth: Optional[Tuple[str, str]] = None,verify: Optional[bool] = _UNSET,cert: Optional[str | Tuple[str, str]] = _UNSET,- impersonate: Optional[BrowserTypeLiteral] = _UNSET,+ impersonate: ImpersonateType = _UNSET,http3: Optional[bool] = _UNSET,stealthy_headers: Optional[bool] = _UNSET,**kwargs,@@ -896,7 +916,7 @@ class _ASyncSessionLogic(_ConfigurationLogic):auth: Optional[Tuple[str, str]] = None,verify: Optional[bool] = _UNSET,cert: Optional[str | Tuple[str, str]] = _UNSET,- impersonate: Optional[BrowserTypeLiteral] = _UNSET,+ impersonate: ImpersonateType = _UNSET,http3: Optional[bool] = _UNSET,stealthy_headers: Optional[bool] = _UNSET,**kwargs,@@ -970,7 +990,7 @@ class FetcherSession:def __init__(self,- impersonate: Optional[BrowserTypeLiteral] = "chrome",+ impersonate: ImpersonateType = "chrome",http3: Optional[bool] = False,stealthy_headers: Optional[bool] = True,proxies: Optional[Dict[str, str]] = None,@@ -987,7 +1007,7 @@ class FetcherSession:selector_config: Optional[Dict] = None,):"""- :param impersonate: Browser version to impersonate. Automatically defaults to the latest available Chrome version.+ :param impersonate: Browser version to impersonate. Can be a single browser string or a list of browser strings for random selection. (Default: latest available Chrome version):param http3: Whether to use HTTP3. Defaults to False. It might be problematic if used it with `impersonate`.:param stealthy_headers: If enabled (default), it creates and adds real browser headers. It also sets the referer header as if this request came from a Google search of URL's domain.:param proxies: Dict of proxies to use. Format: {"http": proxy_url, "https": proxy_url}.@@ -1004,7 +1024,7 @@ class FetcherSession::param cert: Tuple of (cert, key) filenames for the client certificate.:param selector_config: Arguments passed when creating the final Selector class."""- self._default_impersonate: Optional[BrowserTypeLiteral] = impersonate+ self._default_impersonate: ImpersonateType = impersonateself._stealth = stealthy_headersself._default_proxies = proxies or {}self._default_proxy = proxy or None