Skip to content

🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev

fix: make request fingerprint independent of header order (#466)

Sarwadnya Mahajan4b97bfe4a3c4
diff --git a/scrapling/spiders/request.py b/scrapling/spiders/request.py
index 8809c15..0f87546 100644
--- a/scrapling/spiders/request.py
+++ b/scrapling/spiders/request.py
@@ -121,7 +121,7 @@ class Request:
# Some header normalization
for key, value in headers.items():
processed_headers[_convert_to_bytes(key.lower()).hex()] = _convert_to_bytes(value).hex()
- data["headers"] = tuple(processed_headers.items())
+ data["headers"] = tuple(sorted(processed_headers.items()))
fp = hashlib.sha1(orjson.dumps(data, option=orjson.OPT_SORT_KEYS), usedforsecurity=False).digest()
self._fp = fp
diff --git a/tests/spiders/test_request.py b/tests/spiders/test_request.py
index 296b660..e2f3767 100644
--- a/tests/spiders/test_request.py
+++ b/tests/spiders/test_request.py
@@ -131,6 +131,13 @@ class TestRequestProperties:
assert r1.update_fingerprint(include_headers=True) != r2.update_fingerprint(include_headers=True)
+ def test_fingerprint_include_headers_order_independent(self):
+ """Test header order does not affect request fingerprint."""
+ r1 = Request("https://example.com", headers={"Accept": "text/html", "User-Agent": "x"})
+ r2 = Request("https://example.com", headers={"User-Agent": "x", "Accept": "text/html"})
+
+ assert r1.update_fingerprint(include_headers=True) == r2.update_fingerprint(include_headers=True)
+
def test_fingerprint_includes_params(self):
r1 = Request("https://example.com/p?a=1", params={"skip": 1})
r2 = Request("https://example.com/p?a=1", params={"skip": 2})