🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev
Python aiai-scrapingautomationcrawlercrawlingcrawling-pythondatadata-extractionmcpmcp-serverplaywrightpythonscrapingselectorsstealthweb-scraperweb-scrapingweb-scraping-pythonwebscrapingxpath
fix(browsers): solve a bug with setting referer on the request level
Karim shoairc5bc82deb813
diff --git a/scrapling/core/_types.py b/scrapling/core/_types.pyindex c897954..48a2eec 100644--- a/scrapling/core/_types.py+++ b/scrapling/core/_types.py@@ -19,6 +19,7 @@ from typing import (Set,Literal,Optional,+ Iterator,Pattern,Sequence,Tuple,diff --git a/scrapling/engines/_browsers/_controllers.py b/scrapling/engines/_browsers/_controllers.pyindex 285bb32..aef197a 100644--- a/scrapling/engines/_browsers/_controllers.py+++ b/scrapling/engines/_browsers/_controllers.py@@ -108,8 +108,11 @@ class DynamicSession(SyncSession, DynamicSessionMixin):if not self._is_alive: # pragma: no coverraise RuntimeError("Context manager has been closed")+ request_headers_keys = {h.lower() for h in params.extra_headers.keys()} if params.extra_headers else set()referer = (- generate_convincing_referer(url) if (params.google_search and "referer" not in self._headers_keys) else None+ generate_convincing_referer(url)+ if (params.google_search and "referer" not in request_headers_keys)+ else None)page_info = self._get_page(params.timeout, params.extra_headers, params.disable_resources)@@ -240,8 +243,11 @@ class AsyncDynamicSession(AsyncSession, DynamicSessionMixin):if not self._is_alive: # pragma: no coverraise RuntimeError("Context manager has been closed")+ request_headers_keys = {h.lower() for h in params.extra_headers.keys()} if params.extra_headers else set()referer = (- generate_convincing_referer(url) if (params.google_search and "referer" not in self._headers_keys) else None+ generate_convincing_referer(url)+ if (params.google_search and "referer" not in request_headers_keys)+ else None)page_info = await self._get_page(params.timeout, params.extra_headers, params.disable_resources)diff --git a/scrapling/engines/_browsers/_stealth.py b/scrapling/engines/_browsers/_stealth.pyindex e191059..6883fdf 100644--- a/scrapling/engines/_browsers/_stealth.py+++ b/scrapling/engines/_browsers/_stealth.py@@ -195,8 +195,11 @@ class StealthySession(SyncSession, StealthySessionMixin):if not self._is_alive: # pragma: no coverraise RuntimeError("Context manager has been closed")+ request_headers_keys = {h.lower() for h in params.extra_headers.keys()} if params.extra_headers else set()referer = (- generate_convincing_referer(url) if (params.google_search and "referer" not in self._headers_keys) else None+ generate_convincing_referer(url)+ if (params.google_search and "referer" not in request_headers_keys)+ else None)page_info = self._get_page(params.timeout, params.extra_headers, params.disable_resources)@@ -410,8 +413,11 @@ class AsyncStealthySession(AsyncSession, StealthySessionMixin):if not self._is_alive: # pragma: no coverraise RuntimeError("Context manager has been closed")+ request_headers_keys = {h.lower() for h in params.extra_headers.keys()} if params.extra_headers else set()referer = (- generate_convincing_referer(url) if (params.google_search and "referer" not in self._headers_keys) else None+ generate_convincing_referer(url)+ if (params.google_search and "referer" not in request_headers_keys)+ else None)page_info = await self._get_page(params.timeout, params.extra_headers, params.disable_resources)