Skip to content

🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev

fix(fetcher): Fix impersonate and headers generation conflict

Karim shoaird049f63404fc
diff --git a/scrapling/engines/static.py b/scrapling/engines/static.py
index 869d56f..aefec93 100644
--- a/scrapling/engines/static.py
+++ b/scrapling/engines/static.py
@@ -111,11 +111,13 @@ class FetcherSession:
"The argument `http3` might cause errors if used with `impersonate` argument, try switching it off if you encounter any curl errors."
)
+ impersonate = kwargs.pop("impersonate", self.default_impersonate)
request_args.update(
{
"url": url,
+ # Curl automatically generates the suitable browser headers when you use `impersonate`
"headers": self._headers_job(
- url, kwargs.pop("headers"), kwargs.pop("stealth")
+ url, kwargs.pop("headers"), kwargs.pop("stealth"), bool(impersonate)
),
"proxies": kwargs.pop("proxies", self.default_proxies),
"proxy": kwargs.pop("proxy", self.default_proxy),
@@ -129,26 +131,37 @@ class FetcherSession:
),
"verify": kwargs.pop("verify", self.default_verify),
"cert": kwargs.pop("cert", self.default_cert),
- "impersonate": kwargs.pop("impersonate", self.default_impersonate),
+ "impersonate": impersonate,
**kwargs,
}
)
return request_args
def _headers_job(
- self, url, headers: Optional[Dict], stealth: Optional[bool]
+ self,
+ url,
+ headers: Optional[Dict],
+ stealth: Optional[bool],
+ impersonate_enabled: bool,
) -> Dict:
"""Adds useragent to headers if it doesn't exist, generates real headers and append it to current headers, and
finally generates a referer header that looks like if this request came from Google's search of the current URL's domain.
:param headers: Current headers in the request if the user passed any
:param stealth: Whether to enable the `stealthy_headers` argument to this request or not. If `None`, it defaults to the session default value.
+ :param impersonate_enabled: Whether the browser impersonation is enabled or not.
:return: A dictionary of the new headers.
"""
headers = {**self.default_headers, **(headers or {})}
headers_keys = set(map(str.lower, headers.keys()))
if stealth:
+ if "referer" not in headers_keys:
+ headers.update({"referer": generate_convincing_referer(url)})
+
+ if impersonate_enabled: # Curl will generate the suitable headers
+ return headers
+
extra_headers = generate_headers(browser_mode=False)
# Don't overwrite user-supplied headers
extra_headers = {
@@ -157,10 +170,8 @@ class FetcherSession:
if key.lower() not in headers_keys
}
headers.update(extra_headers)
- if "referer" not in headers_keys:
- headers.update({"referer": generate_convincing_referer(url)})
- elif "user-agent" not in headers_keys:
+ elif "user-agent" not in headers_keys and not impersonate_enabled:
headers["User-Agent"] = __default_useragent__
log.debug(
f"Can't find useragent in headers so '{headers['User-Agent']}' was used."