🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev
Python aiai-scrapingautomationcrawlercrawlingcrawling-pythondatadata-extractionmcpmcp-serverplaywrightpythonscrapingselectorsstealthweb-scraperweb-scrapingweb-scraping-pythonwebscrapingxpath
fix(fetchers): Fix the bug of referer and `google_search` argument conflict
Karim shoair78b81c537f95
diff --git a/scrapling/engines/_browsers/_camoufox.py b/scrapling/engines/_browsers/_camoufox.pyindex b7335c0..837bb6d 100644--- a/scrapling/engines/_browsers/_camoufox.py+++ b/scrapling/engines/_browsers/_camoufox.py@@ -83,6 +83,7 @@ class StealthySession:"_closed","launch_options","context_options",+ "_headers_keys",)def __init__(@@ -204,6 +205,11 @@ class StealthySession:self._closed = Falseself.adaptor_arguments = config.adaptor_argumentsself.page_action = config.page_action+ self._headers_keys = (+ set(map(str.lower, self.extra_headers.keys()))+ if self.extra_headers+ else set()+ )self.__initiate_browser_options__()def __initiate_browser_options__(self):@@ -377,7 +383,11 @@ class StealthySession:raise RuntimeError("Context manager has been closed")final_response = None- referer = generate_convincing_referer(url) if self.google_search else None+ referer = (+ generate_convincing_referer(url)+ if (self.google_search and "referer" not in self._headers_keys)+ else None+ )def handle_response(finished_response: SyncPlaywrightResponse):nonlocal final_response@@ -673,7 +683,11 @@ class AsyncStealthySession(StealthySession):raise RuntimeError("Context manager has been closed")final_response = None- referer = generate_convincing_referer(url) if self.google_search else None+ referer = (+ generate_convincing_referer(url)+ if (self.google_search and "referer" not in self._headers_keys)+ else None+ )async def handle_response(finished_response: AsyncPlaywrightResponse):nonlocal final_responsediff --git a/scrapling/engines/_browsers/_controllers.py b/scrapling/engines/_browsers/_controllers.pyindex 5148362..fef4acb 100644--- a/scrapling/engines/_browsers/_controllers.py+++ b/scrapling/engines/_browsers/_controllers.py@@ -77,6 +77,7 @@ class DynamicSession:"launch_options","context_options","cdp_url",+ "_headers_keys",)def __init__(@@ -182,6 +183,11 @@ class DynamicSession:self._closed = Falseself.adaptor_arguments = config.adaptor_argumentsself.page_action = config.page_action+ self._headers_keys = (+ set(map(str.lower, self.extra_headers.keys()))+ if self.extra_headers+ else set()+ )self.__initiate_browser_options__()def __initiate_browser_options__(self):@@ -301,7 +307,11 @@ class DynamicSession:raise RuntimeError("Context manager has been closed")final_response = None- referer = generate_convincing_referer(url) if self.google_search else None+ referer = (+ generate_convincing_referer(url)+ if (self.google_search and "referer" not in self._headers_keys)+ else None+ )def handle_response(finished_response: SyncPlaywrightResponse):nonlocal final_response@@ -554,7 +564,11 @@ class AsyncDynamicSession(DynamicSession):raise RuntimeError("Context manager has been closed")final_response = None- referer = generate_convincing_referer(url) if self.google_search else None+ referer = (+ generate_convincing_referer(url)+ if (self.google_search and "referer" not in self._headers_keys)+ else None+ )async def handle_response(finished_response: AsyncPlaywrightResponse):nonlocal final_response