🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev
Python aiai-scrapingautomationcrawlercrawlingcrawling-pythondatadata-extractionmcpmcp-serverplaywrightpythonscrapingselectorsstealthweb-scraperweb-scrapingweb-scraping-pythonwebscrapingxpath
fix: forgot to stage it with last commit
Karim shoaire25434197469
diff --git a/scrapling/engines/static.py b/scrapling/engines/static.pyindex 9bad034..557c6b9 100644--- a/scrapling/engines/static.py+++ b/scrapling/engines/static.py@@ -2,6 +2,7 @@ import httpxfrom httpx._models import Response as httpxResponsefrom scrapling.core._types import Dict, Optional, Union+from scrapling.core.utils import logfrom .toolbelt import Response, generate_convincing_referer, generate_headers@@ -34,6 +35,7 @@ class StaticEngine:# Validate headersif not headers.get('user-agent') and not headers.get('User-Agent'):headers['User-Agent'] = generate_headers(browser_mode=False).get('User-Agent')+ log.debug(f"Can't find useragent in headers so '{headers['User-Agent']}' was used.")if stealth:extra_headers = generate_headers(browser_mode=False)@@ -58,6 +60,7 @@ class StaticEngine:cookies=dict(response.cookies),headers=dict(response.headers),request_headers=dict(response.request.headers),+ method=response.request.method,**self.adaptor_arguments)