🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev
Python aiai-scrapingautomationcrawlercrawlingcrawling-pythondatadata-extractionmcpmcp-serverplaywrightpythonscrapingselectorsstealthweb-scraperweb-scrapingweb-scraping-pythonwebscrapingxpath
fix: make request fingerprint independent of header order (#466)
Sarwadnya Mahajan4b97bfe4a3c4
diff --git a/scrapling/spiders/request.py b/scrapling/spiders/request.pyindex 8809c15..0f87546 100644--- a/scrapling/spiders/request.py+++ b/scrapling/spiders/request.py@@ -121,7 +121,7 @@ class Request:# Some header normalizationfor key, value in headers.items():processed_headers[_convert_to_bytes(key.lower()).hex()] = _convert_to_bytes(value).hex()- data["headers"] = tuple(processed_headers.items())+ data["headers"] = tuple(sorted(processed_headers.items()))fp = hashlib.sha1(orjson.dumps(data, option=orjson.OPT_SORT_KEYS), usedforsecurity=False).digest()self._fp = fpdiff --git a/tests/spiders/test_request.py b/tests/spiders/test_request.pyindex 296b660..e2f3767 100644--- a/tests/spiders/test_request.py+++ b/tests/spiders/test_request.py@@ -131,6 +131,13 @@ class TestRequestProperties:assert r1.update_fingerprint(include_headers=True) != r2.update_fingerprint(include_headers=True)+ def test_fingerprint_include_headers_order_independent(self):+ """Test header order does not affect request fingerprint."""+ r1 = Request("https://example.com", headers={"Accept": "text/html", "User-Agent": "x"})+ r2 = Request("https://example.com", headers={"User-Agent": "x", "Accept": "text/html"})++ assert r1.update_fingerprint(include_headers=True) == r2.update_fingerprint(include_headers=True)+def test_fingerprint_includes_params(self):r1 = Request("https://example.com/p?a=1", params={"skip": 1})r2 = Request("https://example.com/p?a=1", params={"skip": 2})