🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev
Python aiai-scrapingautomationcrawlercrawlingcrawling-pythondatadata-extractionmcpmcp-serverplaywrightpythonscrapingselectorsstealthweb-scraperweb-scrapingweb-scraping-pythonwebscrapingxpath
fix(Fetcher): make `stealthy_headers` don't overwrite user headers
Karim shoair5f4ec3ba3a5c
diff --git a/scrapling/engines/static.py b/scrapling/engines/static.pyindex a939c46..a58aa39 100644--- a/scrapling/engines/static.py+++ b/scrapling/engines/static.py@@ -42,16 +42,19 @@ class StaticEngine::return: A dictionary of the new headers."""headers = headers or {}-- # Validate headers- if not headers.get('user-agent') and not headers.get('User-Agent'):- headers['User-Agent'] = generate_headers(browser_mode=False).get('User-Agent')- log.debug(f"Can't find useragent in headers so '{headers['User-Agent']}' was used.")+ headers_keys = set(map(str.lower, headers.keys()))if self.stealth:extra_headers = generate_headers(browser_mode=False)+ # Don't overwrite user supplied headers+ extra_headers = {key: value for key, value in extra_headers.items() if key.lower() not in headers_keys}headers.update(extra_headers)- headers.update({'referer': generate_convincing_referer(self.url)})+ if 'referer' not in headers_keys:+ headers.update({'referer': generate_convincing_referer(self.url)})++ elif 'user-agent' not in headers_keys:+ headers['User-Agent'] = generate_headers(browser_mode=False).get('User-Agent')+ log.debug(f"Can't find useragent in headers so '{headers['User-Agent']}' was used.")return headers