🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev
Python aiai-scrapingautomationcrawlercrawlingcrawling-pythondatadata-extractionmcpmcp-serverplaywrightpythonscrapingselectorsstealthweb-scraperweb-scrapingweb-scraping-pythonwebscrapingxpath
fix(spiders): include params in the request fingerprint (#448)
Yuval Dinodia2beadcf7af00
diff --git a/scrapling/spiders/request.py b/scrapling/spiders/request.pyindex ef766b2..8809c15 100644--- a/scrapling/spiders/request.py+++ b/scrapling/spiders/request.py@@ -4,6 +4,7 @@ from functools import cached_propertyfrom urllib.parse import urlparse, urlencodeimport orjson+from curl_cffi.requests.utils import update_url_paramsfrom w3lib.url import canonicalize_urlfrom scrapling.engines.toolbelt.custom import Response@@ -96,11 +97,14 @@ class Request:post_data = self._session_kwargs.get("json", {})body = orjson.dumps(post_data) if post_data else b""+ params = self._session_kwargs.get("params")+ url = update_url_params(self.url, params) if params else self.url+data: Dict[str, str | Tuple] = {"sid": self.sid,"body": body.hex(),"method": self._session_kwargs.get("method", "GET"),- "url": canonicalize_url(self.url, keep_fragments=keep_fragments),+ "url": canonicalize_url(url, keep_fragments=keep_fragments),}if include_kwargs:diff --git a/tests/spiders/test_request.py b/tests/spiders/test_request.pyindex 00f6c49..296b660 100644--- a/tests/spiders/test_request.py+++ b/tests/spiders/test_request.py@@ -131,6 +131,34 @@ class TestRequestProperties:assert r1.update_fingerprint(include_headers=True) != r2.update_fingerprint(include_headers=True)+ def test_fingerprint_includes_params(self):+ r1 = Request("https://example.com/p?a=1", params={"skip": 1})+ r2 = Request("https://example.com/p?a=1", params={"skip": 2})+ r3 = Request("https://example.com/p?skip=2&a=1")++ assert r1.update_fingerprint() != r2.update_fingerprint()+ assert r2.update_fingerprint() == r3.update_fingerprint()++ def test_fingerprint_params_override_existing_query_key(self):+ r1 = Request("https://example.com/p?skip=0", params={"skip": 1})+ r2 = Request("https://example.com/p?skip=1")++ assert r1.update_fingerprint() == r2.update_fingerprint()++ def test_fingerprint_params_keep_fragment(self):+ r1 = Request("https://example.com/p#top", params={"skip": 1})+ r2 = Request("https://example.com/p?skip=1#top")+ r3 = Request("https://example.com/p#top")++ assert r1.update_fingerprint(keep_fragments=True) == r2.update_fingerprint(keep_fragments=True)+ assert r1.update_fingerprint() != r3.update_fingerprint()++ def test_fingerprint_params_bool_and_list_values(self):+ r1 = Request("https://example.com/p", params={"flag": True, "tag": ["x", "y"]})+ r2 = Request("https://example.com/p?flag=true&tag=x&tag=y")++ assert r1.update_fingerprint() == r2.update_fingerprint()+class TestRequestCopy:"""Test Request copy functionality."""