Skip to content

🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev

feat(fetcher): Make `impersonate` able to randomize fingerprint

Karim shoair3565f9d5009b
diff --git a/scrapling/cli.py b/scrapling/cli.py
index 25fdce3..22f2dd1 100644
--- a/scrapling/cli.py
+++ b/scrapling/cli.py
@@ -2,6 +2,8 @@ from pathlib import Path
from subprocess import check_output
from sys import executable as python_executable
+from curl_cffi.requests import impersonate
+
from scrapling.core.utils import log
from scrapling.engines.toolbelt.custom import Response
from scrapling.core.utils._shell import _CookieParser, _ParseHeaders
@@ -84,7 +86,7 @@ def __BuildRequest(headers: List[str], cookies: str, params: str, json: Optional
# Parse parameters
parsed_headers, parsed_cookies, parsed_params, parsed_json = __ParseExtractArguments(headers, cookies, params, json)
# Build request arguments
- request_kwargs = {
+ request_kwargs: Dict[str, Any] = {
"headers": parsed_headers if parsed_headers else None,
"cookies": parsed_cookies if parsed_cookies else None,
}
@@ -95,6 +97,10 @@ def __BuildRequest(headers: List[str], cookies: str, params: str, json: Optional
if "proxy" in kwargs:
request_kwargs["proxy"] = kwargs.pop("proxy")
+ # Parse impersonate parameter if it contains commas (for random selection)
+ if "impersonate" in kwargs and "," in (kwargs.get("impersonate") or ""):
+ kwargs["impersonate"] = [browser.strip() for browser in kwargs["impersonate"].split(",")]
+
return {**request_kwargs, **kwargs}
@@ -225,7 +231,10 @@ def extract():
default=True,
help="Whether to verify SSL certificates (default: True)",
)
-@option("--impersonate", help="Browser to impersonate (e.g., chrome, firefox).")
+@option(
+ "--impersonate",
+ help="Browser to impersonate. Can be a single browser (e.g., chrome) or comma-separated list for random selection (e.g., chrome,firefox,safari).",
+)
@option(
"--stealthy-headers/--no-stealthy-headers",
default=True,
@@ -318,7 +327,10 @@ def get(
default=True,
help="Whether to verify SSL certificates (default: True)",
)
-@option("--impersonate", help="Browser to impersonate (e.g., chrome, firefox).")
+@option(
+ "--impersonate",
+ help="Browser to impersonate. Can be a single browser (e.g., chrome) or comma-separated list for random selection (e.g., chrome,firefox,safari).",
+)
@option(
"--stealthy-headers/--no-stealthy-headers",
default=True,
@@ -412,7 +424,10 @@ def post(
default=True,
help="Whether to verify SSL certificates (default: True)",
)
-@option("--impersonate", help="Browser to impersonate (e.g., chrome, firefox).")
+@option(
+ "--impersonate",
+ help="Browser to impersonate. Can be a single browser (e.g., chrome) or comma-separated list for random selection (e.g., chrome,firefox,safari).",
+)
@option(
"--stealthy-headers/--no-stealthy-headers",
default=True,
@@ -504,7 +519,10 @@ def put(
default=True,
help="Whether to verify SSL certificates (default: True)",
)
-@option("--impersonate", help="Browser to impersonate (e.g., chrome, firefox).")
+@option(
+ "--impersonate",
+ help="Browser to impersonate. Can be a single browser (e.g., chrome) or comma-separated list for random selection (e.g., chrome,firefox,safari).",
+)
@option(
"--stealthy-headers/--no-stealthy-headers",
default=True,
diff --git a/scrapling/core/ai.py b/scrapling/core/ai.py
index d497e57..4d5277a 100644
--- a/scrapling/core/ai.py
+++ b/scrapling/core/ai.py
@@ -5,6 +5,7 @@ from pydantic import BaseModel, Field
from scrapling.core.shell import Convertor
from scrapling.engines.toolbelt.custom import Response as _ScraplingResponse
+from scrapling.engines.static import ImpersonateType
from scrapling.fetchers import (
Fetcher,
FetcherSession,
@@ -24,9 +25,6 @@ from scrapling.core._types import (
SelectorWaitStates,
Generator,
)
-from curl_cffi.requests import (
- BrowserTypeLiteral,
-)
class ResponseModel(BaseModel):
@@ -46,7 +44,7 @@ class ScraplingMCPServer:
@staticmethod
def get(
url: str,
- impersonate: Optional[BrowserTypeLiteral] = "chrome",
+ impersonate: ImpersonateType = "chrome",
extraction_type: extraction_types = "markdown",
css_selector: Optional[str] = None,
main_content_only: bool = True,
@@ -124,7 +122,7 @@ class ScraplingMCPServer:
@staticmethod
async def bulk_get(
urls: Tuple[str, ...],
- impersonate: Optional[BrowserTypeLiteral] = "chrome",
+ impersonate: ImpersonateType = "chrome",
extraction_type: extraction_types = "markdown",
css_selector: Optional[str] = None,
main_content_only: bool = True,
diff --git a/scrapling/engines/static.py b/scrapling/engines/static.py
index dd54168..6f68737 100644
--- a/scrapling/engines/static.py
+++ b/scrapling/engines/static.py
@@ -1,4 +1,5 @@
from abc import ABC
+from random import choice
from time import sleep as time_sleep
from asyncio import sleep as asyncio_sleep
@@ -31,12 +32,29 @@ from .toolbelt.fingerprints import generate_convincing_referer, generate_headers
_UNSET: Any = object()
_NO_SESSION: Any = object()
+# Type alias for `impersonate` parameter - accepts a single browser or list of browsers
+ImpersonateType = BrowserTypeLiteral | List[BrowserTypeLiteral] | None
+
+
+def _select_random_browser(impersonate: ImpersonateType) -> Optional[BrowserTypeLiteral]:
+ """
+ Handle browser selection logic for the ` impersonate ` parameter.
+
+ If impersonate is a list, randomly select one browser from it.
+ If it's a string or None, return as is.
+ """
+ if isinstance(impersonate, list):
+ if not impersonate:
+ return None
+ return choice(impersonate)
+ return impersonate
+
class _ConfigurationLogic(ABC):
# Core Logic Handler (Internal Engine)
def __init__(
self,
- impersonate: Optional[BrowserTypeLiteral] = "chrome",
+ impersonate: ImpersonateType = "chrome",
http3: Optional[bool] = False,
stealthy_headers: Optional[bool] = True,
proxies: Optional[Dict[str, str]] = None,
@@ -76,7 +94,9 @@ class _ConfigurationLogic(ABC):
def _merge_request_args(self, **method_kwargs) -> Dict[str, Any]:
"""Merge request-specific arguments with default session arguments."""
url = method_kwargs.pop("url")
- impersonate = self._get_with_precedence(method_kwargs.pop("impersonate"), self._default_impersonate)
+ impersonate = _select_random_browser(
+ self._get_with_precedence(method_kwargs.pop("impersonate"), self._default_impersonate)
+ )
http3_enabled = self._get_with_precedence(method_kwargs.pop("http3"), self._default_http3)
final_args = {
"url": url,
@@ -146,7 +166,7 @@ class _ConfigurationLogic(ABC):
class _SyncSessionLogic(_ConfigurationLogic):
def __init__(
self,
- impersonate: Optional[BrowserTypeLiteral] = "chrome",
+ impersonate: ImpersonateType = "chrome",
http3: Optional[bool] = False,
stealthy_headers: Optional[bool] = True,
proxies: Optional[Dict[str, str]] = None,
@@ -261,7 +281,7 @@ class _SyncSessionLogic(_ConfigurationLogic):
auth: Optional[Tuple[str, str]] = None,
verify: Optional[bool] = _UNSET,
cert: Optional[str | Tuple[str, str]] = _UNSET,
- impersonate: Optional[BrowserTypeLiteral] = _UNSET,
+ impersonate: ImpersonateType = _UNSET,
http3: Optional[bool] = _UNSET,
stealthy_headers: Optional[bool] = _UNSET,
**kwargs,
@@ -334,7 +354,7 @@ class _SyncSessionLogic(_ConfigurationLogic):
auth: Optional[Tuple[str, str]] = None,
verify: Optional[bool] = _UNSET,
cert: Optional[str | Tuple[str, str]] = _UNSET,
- impersonate: Optional[BrowserTypeLiteral] = _UNSET,
+ impersonate: ImpersonateType = _UNSET,
http3: Optional[bool] = _UNSET,
stealthy_headers: Optional[bool] = _UNSET,
**kwargs,
@@ -411,7 +431,7 @@ class _SyncSessionLogic(_ConfigurationLogic):
auth: Optional[Tuple[str, str]] = None,
verify: Optional[bool] = _UNSET,
cert: Optional[str | Tuple[str, str]] = _UNSET,
- impersonate: Optional[BrowserTypeLiteral] = _UNSET,
+ impersonate: ImpersonateType = _UNSET,
http3: Optional[bool] = _UNSET,
stealthy_headers: Optional[bool] = _UNSET,
**kwargs,
@@ -488,7 +508,7 @@ class _SyncSessionLogic(_ConfigurationLogic):
auth: Optional[Tuple[str, str]] = None,
verify: Optional[bool] = _UNSET,
cert: Optional[str | Tuple[str, str]] = _UNSET,
- impersonate: Optional[BrowserTypeLiteral] = _UNSET,
+ impersonate: ImpersonateType = _UNSET,
http3: Optional[bool] = _UNSET,
stealthy_headers: Optional[bool] = _UNSET,
**kwargs,
@@ -552,7 +572,7 @@ class _SyncSessionLogic(_ConfigurationLogic):
class _ASyncSessionLogic(_ConfigurationLogic):
def __init__(
self,
- impersonate: Optional[BrowserTypeLiteral] = "chrome",
+ impersonate: ImpersonateType = "chrome",
http3: Optional[bool] = False,
stealthy_headers: Optional[bool] = True,
proxies: Optional[Dict[str, str]] = None,
@@ -669,7 +689,7 @@ class _ASyncSessionLogic(_ConfigurationLogic):
auth: Optional[Tuple[str, str]] = None,
verify: Optional[bool] = _UNSET,
cert: Optional[str | Tuple[str, str]] = _UNSET,
- impersonate: Optional[BrowserTypeLiteral] = _UNSET,
+ impersonate: ImpersonateType = _UNSET,
http3: Optional[bool] = _UNSET,
stealthy_headers: Optional[bool] = _UNSET,
**kwargs,
@@ -742,7 +762,7 @@ class _ASyncSessionLogic(_ConfigurationLogic):
auth: Optional[Tuple[str, str]] = None,
verify: Optional[bool] = _UNSET,
cert: Optional[str | Tuple[str, str]] = _UNSET,
- impersonate: Optional[BrowserTypeLiteral] = _UNSET,
+ impersonate: ImpersonateType = _UNSET,
http3: Optional[bool] = _UNSET,
stealthy_headers: Optional[bool] = _UNSET,
**kwargs,
@@ -819,7 +839,7 @@ class _ASyncSessionLogic(_ConfigurationLogic):
auth: Optional[Tuple[str, str]] = None,
verify: Optional[bool] = _UNSET,
cert: Optional[str | Tuple[str, str]] = _UNSET,
- impersonate: Optional[BrowserTypeLiteral] = _UNSET,
+ impersonate: ImpersonateType = _UNSET,
http3: Optional[bool] = _UNSET,
stealthy_headers: Optional[bool] = _UNSET,
**kwargs,
@@ -896,7 +916,7 @@ class _ASyncSessionLogic(_ConfigurationLogic):
auth: Optional[Tuple[str, str]] = None,
verify: Optional[bool] = _UNSET,
cert: Optional[str | Tuple[str, str]] = _UNSET,
- impersonate: Optional[BrowserTypeLiteral] = _UNSET,
+ impersonate: ImpersonateType = _UNSET,
http3: Optional[bool] = _UNSET,
stealthy_headers: Optional[bool] = _UNSET,
**kwargs,
@@ -970,7 +990,7 @@ class FetcherSession:
def __init__(
self,
- impersonate: Optional[BrowserTypeLiteral] = "chrome",
+ impersonate: ImpersonateType = "chrome",
http3: Optional[bool] = False,
stealthy_headers: Optional[bool] = True,
proxies: Optional[Dict[str, str]] = None,
@@ -987,7 +1007,7 @@ class FetcherSession:
selector_config: Optional[Dict] = None,
):
"""
- :param impersonate: Browser version to impersonate. Automatically defaults to the latest available Chrome version.
+ :param impersonate: Browser version to impersonate. Can be a single browser string or a list of browser strings for random selection. (Default: latest available Chrome version)
:param http3: Whether to use HTTP3. Defaults to False. It might be problematic if used it with `impersonate`.
:param stealthy_headers: If enabled (default), it creates and adds real browser headers. It also sets the referer header as if this request came from a Google search of URL's domain.
:param proxies: Dict of proxies to use. Format: {"http": proxy_url, "https": proxy_url}.
@@ -1004,7 +1024,7 @@ class FetcherSession:
:param cert: Tuple of (cert, key) filenames for the client certificate.
:param selector_config: Arguments passed when creating the final Selector class.
"""
- self._default_impersonate: Optional[BrowserTypeLiteral] = impersonate
+ self._default_impersonate: ImpersonateType = impersonate
self._stealth = stealthy_headers
self._default_proxies = proxies or {}
self._default_proxy = proxy or None