🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev
Python aiai-scrapingautomationcrawlercrawlingcrawling-pythondatadata-extractionmcpmcp-serverplaywrightpythonscrapingselectorsstealthweb-scraperweb-scrapingweb-scraping-pythonwebscrapingxpath
fix(fetchers): let request headers override session headers regardless of case (#487)
Suliman Abdulrazzaq71b57943fd08
diff --git a/scrapling/engines/static.py b/scrapling/engines/static.pyindex 91b81ef..1c14d26 100644--- a/scrapling/engines/static.py+++ b/scrapling/engines/static.py@@ -172,7 +172,13 @@ class _ConfigurationLogic(ABC):3. Sets a Google referer header."""# Merge session headers with request headers, request takes precedence (if it was set)- final_headers = {**self._default_headers, **(headers if headers else {})}+ # Header names are case-insensitive, so a session header spelled differently must not be sent next to it+ headers = headers if headers else {}+ request_keys = {k.lower() for k in headers}+ session_headers = {+ k: v for k, v in self._default_headers.items() if k in headers or k.lower() not in request_keys+ }+ final_headers = {**session_headers, **headers}headers_keys = {k.lower() for k in final_headers}if stealth:if "referer" not in headers_keys:diff --git a/tests/fetchers/sync/test_requests_session.py b/tests/fetchers/sync/test_requests_session.pyindex fde37a3..c267374 100644--- a/tests/fetchers/sync/test_requests_session.py+++ b/tests/fetchers/sync/test_requests_session.py@@ -1,3 +1,7 @@+import json+import threading+from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer+import pytestfrom unittest.mock import patch, MagicMockfrom curl_cffi.curl import CurlError@@ -105,3 +109,49 @@ class TestFetcherSession:session.get("http://example.com", retries=0)assert mocked_request.call_count == 1++ def test_request_headers_replace_session_headers_regardless_of_case(self):+ """The server must receive one value per header, the per-request one, whatever the case"""++ class EchoHeaders(BaseHTTPRequestHandler):+ def do_GET(self):+ body = json.dumps(+ {+ "user_agent": self.headers.get_all("User-Agent") or [],+ "authorization": self.headers.get_all("Authorization") or [],+ "accept": self.headers.get_all("Accept") or [],+ }+ ).encode()+ self.send_response(200)+ self.send_header("Content-Type", "application/json")+ self.send_header("Content-Length", str(len(body)))+ self.end_headers()+ self.wfile.write(body)++ def log_message(self, *args):+ pass++ server = ThreadingHTTPServer(("127.0.0.1", 0), EchoHeaders)+ thread = threading.Thread(target=server.serve_forever, daemon=True)+ thread.start()+ url = f"http://127.0.0.1:{server.server_address[1]}/"+ session_headers = {"User-Agent": "SessionUA/1.0", "Authorization": "Bearer SESSION", "Accept": "text/plain"}+ try:+ with FetcherSession(headers=session_headers) as session:+ lowercase = session.get(url, headers={"user-agent": "RequestUA/2.0", "authorization": "Bearer REQUEST"})+ same_case = session.get(url, headers={"User-Agent": "RequestUA/2.0"})+ finally:+ server.shutdown()+ server.server_close()+ thread.join()++ assert lowercase.json() == {+ "user_agent": ["RequestUA/2.0"],+ "authorization": ["Bearer REQUEST"],+ "accept": ["text/plain"],+ }+ assert same_case.json() == {+ "user_agent": ["RequestUA/2.0"],+ "authorization": ["Bearer SESSION"],+ "accept": ["text/plain"],+ }diff --git a/tests/fetchers/test_merge_request_args.py b/tests/fetchers/test_merge_request_args.pyindex 6b1e101..d9185c3 100644--- a/tests/fetchers/test_merge_request_args.py+++ b/tests/fetchers/test_merge_request_args.py@@ -8,6 +8,60 @@ import pytestfrom scrapling.engines.static import FetcherClient+class TestHeadersMergeIsCaseInsensitive:+ """A per-request header replaces a session header whose name only differs by case."""++ URL = "https://example.com"++ def _merge(self, session_headers, request_headers, **extra):+ client = FetcherClient(headers=session_headers, **extra)+ return client._headers_job(self.URL, request_headers, False, True)++ def test_lowercase_request_header_replaces_session_header(self):+ headers = self._merge(+ {"User-Agent": "SessionUA/1.0", "Authorization": "Bearer SESSION"},+ {"user-agent": "RequestUA/2.0", "authorization": "Bearer REQUEST"},+ )+ assert headers == {"user-agent": "RequestUA/2.0", "authorization": "Bearer REQUEST"}++ def test_uppercase_request_header_replaces_lowercase_session_header(self):+ headers = self._merge({"x-token": "session"}, {"X-Token": "request"})+ assert headers == {"X-Token": "request"}++ def test_same_case_override_still_works(self):+ headers = self._merge({"User-Agent": "SessionUA/1.0"}, {"User-Agent": "RequestUA/2.0"})+ assert headers == {"User-Agent": "RequestUA/2.0"}++ def test_session_headers_not_overridden_are_kept_in_order(self):+ headers = self._merge(+ {"Accept": "text/html", "User-Agent": "SessionUA/1.0", "X-Trace": "1"},+ {"user-agent": "RequestUA/2.0", "X-New": "2"},+ )+ assert list(headers.items()) == [+ ("Accept", "text/html"),+ ("X-Trace", "1"),+ ("user-agent", "RequestUA/2.0"),+ ("X-New", "2"),+ ]++ def test_mixed_case_duplicates_inside_request_are_left_alone(self):+ headers = self._merge({}, {"X-Dup": "a", "x-dup": "b"})+ assert headers == {"X-Dup": "a", "x-dup": "b"}++ def test_stealth_does_not_overwrite_user_header_of_any_case(self):+ client = FetcherClient(headers={"Accept-Language": "fr"})+ headers = client._headers_job(+ self.URL, {"user-agent": "RequestUA/2.0", "REFERER": "https://a.example/"}, True, False+ )+ names = [k.lower() for k in headers]+ assert names.count("user-agent") == 1+ assert names.count("accept-language") == 1+ assert names.count("referer") == 1+ assert headers["user-agent"] == "RequestUA/2.0"+ assert headers["Accept-Language"] == "fr"+ assert headers["REFERER"] == "https://a.example/"++class TestMergeRequestArgsSkipsBrowserParams:"""Verify that browser-only keyword arguments are stripped beforethe request dict is forwarded to curl_cffi's Session.request()."""