Skip to content

🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev

fix(fetchers): let request headers override session headers regardless of case (#487)

Suliman Abdulrazzaq71b57943fd08
diff --git a/scrapling/engines/static.py b/scrapling/engines/static.py
index 91b81ef..1c14d26 100644
--- a/scrapling/engines/static.py
+++ b/scrapling/engines/static.py
@@ -172,7 +172,13 @@ class _ConfigurationLogic(ABC):
3. Sets a Google referer header.
"""
# Merge session headers with request headers, request takes precedence (if it was set)
- final_headers = {**self._default_headers, **(headers if headers else {})}
+ # Header names are case-insensitive, so a session header spelled differently must not be sent next to it
+ headers = headers if headers else {}
+ request_keys = {k.lower() for k in headers}
+ session_headers = {
+ k: v for k, v in self._default_headers.items() if k in headers or k.lower() not in request_keys
+ }
+ final_headers = {**session_headers, **headers}
headers_keys = {k.lower() for k in final_headers}
if stealth:
if "referer" not in headers_keys:
diff --git a/tests/fetchers/sync/test_requests_session.py b/tests/fetchers/sync/test_requests_session.py
index fde37a3..c267374 100644
--- a/tests/fetchers/sync/test_requests_session.py
+++ b/tests/fetchers/sync/test_requests_session.py
@@ -1,3 +1,7 @@
+import json
+import threading
+from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
+
import pytest
from unittest.mock import patch, MagicMock
from curl_cffi.curl import CurlError
@@ -105,3 +109,49 @@ class TestFetcherSession:
session.get("http://example.com", retries=0)
assert mocked_request.call_count == 1
+
+ def test_request_headers_replace_session_headers_regardless_of_case(self):
+ """The server must receive one value per header, the per-request one, whatever the case"""
+
+ class EchoHeaders(BaseHTTPRequestHandler):
+ def do_GET(self):
+ body = json.dumps(
+ {
+ "user_agent": self.headers.get_all("User-Agent") or [],
+ "authorization": self.headers.get_all("Authorization") or [],
+ "accept": self.headers.get_all("Accept") or [],
+ }
+ ).encode()
+ self.send_response(200)
+ self.send_header("Content-Type", "application/json")
+ self.send_header("Content-Length", str(len(body)))
+ self.end_headers()
+ self.wfile.write(body)
+
+ def log_message(self, *args):
+ pass
+
+ server = ThreadingHTTPServer(("127.0.0.1", 0), EchoHeaders)
+ thread = threading.Thread(target=server.serve_forever, daemon=True)
+ thread.start()
+ url = f"http://127.0.0.1:{server.server_address[1]}/"
+ session_headers = {"User-Agent": "SessionUA/1.0", "Authorization": "Bearer SESSION", "Accept": "text/plain"}
+ try:
+ with FetcherSession(headers=session_headers) as session:
+ lowercase = session.get(url, headers={"user-agent": "RequestUA/2.0", "authorization": "Bearer REQUEST"})
+ same_case = session.get(url, headers={"User-Agent": "RequestUA/2.0"})
+ finally:
+ server.shutdown()
+ server.server_close()
+ thread.join()
+
+ assert lowercase.json() == {
+ "user_agent": ["RequestUA/2.0"],
+ "authorization": ["Bearer REQUEST"],
+ "accept": ["text/plain"],
+ }
+ assert same_case.json() == {
+ "user_agent": ["RequestUA/2.0"],
+ "authorization": ["Bearer SESSION"],
+ "accept": ["text/plain"],
+ }
diff --git a/tests/fetchers/test_merge_request_args.py b/tests/fetchers/test_merge_request_args.py
index 6b1e101..d9185c3 100644
--- a/tests/fetchers/test_merge_request_args.py
+++ b/tests/fetchers/test_merge_request_args.py
@@ -8,6 +8,60 @@ import pytest
from scrapling.engines.static import FetcherClient
+class TestHeadersMergeIsCaseInsensitive:
+ """A per-request header replaces a session header whose name only differs by case."""
+
+ URL = "https://example.com"
+
+ def _merge(self, session_headers, request_headers, **extra):
+ client = FetcherClient(headers=session_headers, **extra)
+ return client._headers_job(self.URL, request_headers, False, True)
+
+ def test_lowercase_request_header_replaces_session_header(self):
+ headers = self._merge(
+ {"User-Agent": "SessionUA/1.0", "Authorization": "Bearer SESSION"},
+ {"user-agent": "RequestUA/2.0", "authorization": "Bearer REQUEST"},
+ )
+ assert headers == {"user-agent": "RequestUA/2.0", "authorization": "Bearer REQUEST"}
+
+ def test_uppercase_request_header_replaces_lowercase_session_header(self):
+ headers = self._merge({"x-token": "session"}, {"X-Token": "request"})
+ assert headers == {"X-Token": "request"}
+
+ def test_same_case_override_still_works(self):
+ headers = self._merge({"User-Agent": "SessionUA/1.0"}, {"User-Agent": "RequestUA/2.0"})
+ assert headers == {"User-Agent": "RequestUA/2.0"}
+
+ def test_session_headers_not_overridden_are_kept_in_order(self):
+ headers = self._merge(
+ {"Accept": "text/html", "User-Agent": "SessionUA/1.0", "X-Trace": "1"},
+ {"user-agent": "RequestUA/2.0", "X-New": "2"},
+ )
+ assert list(headers.items()) == [
+ ("Accept", "text/html"),
+ ("X-Trace", "1"),
+ ("user-agent", "RequestUA/2.0"),
+ ("X-New", "2"),
+ ]
+
+ def test_mixed_case_duplicates_inside_request_are_left_alone(self):
+ headers = self._merge({}, {"X-Dup": "a", "x-dup": "b"})
+ assert headers == {"X-Dup": "a", "x-dup": "b"}
+
+ def test_stealth_does_not_overwrite_user_header_of_any_case(self):
+ client = FetcherClient(headers={"Accept-Language": "fr"})
+ headers = client._headers_job(
+ self.URL, {"user-agent": "RequestUA/2.0", "REFERER": "https://a.example/"}, True, False
+ )
+ names = [k.lower() for k in headers]
+ assert names.count("user-agent") == 1
+ assert names.count("accept-language") == 1
+ assert names.count("referer") == 1
+ assert headers["user-agent"] == "RequestUA/2.0"
+ assert headers["Accept-Language"] == "fr"
+ assert headers["REFERER"] == "https://a.example/"
+
+
class TestMergeRequestArgsSkipsBrowserParams:
"""Verify that browser-only keyword arguments are stripped before
the request dict is forwarded to curl_cffi's Session.request()."""