Skip to content

🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev

fix(fetchers): Fix the bug of referer and `google_search` argument conflict

Karim shoair78b81c537f95
diff --git a/scrapling/engines/_browsers/_camoufox.py b/scrapling/engines/_browsers/_camoufox.py
index b7335c0..837bb6d 100644
--- a/scrapling/engines/_browsers/_camoufox.py
+++ b/scrapling/engines/_browsers/_camoufox.py
@@ -83,6 +83,7 @@ class StealthySession:
"_closed",
"launch_options",
"context_options",
+ "_headers_keys",
)
def __init__(
@@ -204,6 +205,11 @@ class StealthySession:
self._closed = False
self.adaptor_arguments = config.adaptor_arguments
self.page_action = config.page_action
+ self._headers_keys = (
+ set(map(str.lower, self.extra_headers.keys()))
+ if self.extra_headers
+ else set()
+ )
self.__initiate_browser_options__()
def __initiate_browser_options__(self):
@@ -377,7 +383,11 @@ class StealthySession:
raise RuntimeError("Context manager has been closed")
final_response = None
- referer = generate_convincing_referer(url) if self.google_search else None
+ referer = (
+ generate_convincing_referer(url)
+ if (self.google_search and "referer" not in self._headers_keys)
+ else None
+ )
def handle_response(finished_response: SyncPlaywrightResponse):
nonlocal final_response
@@ -673,7 +683,11 @@ class AsyncStealthySession(StealthySession):
raise RuntimeError("Context manager has been closed")
final_response = None
- referer = generate_convincing_referer(url) if self.google_search else None
+ referer = (
+ generate_convincing_referer(url)
+ if (self.google_search and "referer" not in self._headers_keys)
+ else None
+ )
async def handle_response(finished_response: AsyncPlaywrightResponse):
nonlocal final_response
diff --git a/scrapling/engines/_browsers/_controllers.py b/scrapling/engines/_browsers/_controllers.py
index 5148362..fef4acb 100644
--- a/scrapling/engines/_browsers/_controllers.py
+++ b/scrapling/engines/_browsers/_controllers.py
@@ -77,6 +77,7 @@ class DynamicSession:
"launch_options",
"context_options",
"cdp_url",
+ "_headers_keys",
)
def __init__(
@@ -182,6 +183,11 @@ class DynamicSession:
self._closed = False
self.adaptor_arguments = config.adaptor_arguments
self.page_action = config.page_action
+ self._headers_keys = (
+ set(map(str.lower, self.extra_headers.keys()))
+ if self.extra_headers
+ else set()
+ )
self.__initiate_browser_options__()
def __initiate_browser_options__(self):
@@ -301,7 +307,11 @@ class DynamicSession:
raise RuntimeError("Context manager has been closed")
final_response = None
- referer = generate_convincing_referer(url) if self.google_search else None
+ referer = (
+ generate_convincing_referer(url)
+ if (self.google_search and "referer" not in self._headers_keys)
+ else None
+ )
def handle_response(finished_response: SyncPlaywrightResponse):
nonlocal final_response
@@ -554,7 +564,11 @@ class AsyncDynamicSession(DynamicSession):
raise RuntimeError("Context manager has been closed")
final_response = None
- referer = generate_convincing_referer(url) if self.google_search else None
+ referer = (
+ generate_convincing_referer(url)
+ if (self.google_search and "referer" not in self._headers_keys)
+ else None
+ )
async def handle_response(finished_response: AsyncPlaywrightResponse):
nonlocal final_response