Skip to content

🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev

fix(spiders): include params in the request fingerprint (#448)

Yuval Dinodia2beadcf7af00
diff --git a/scrapling/spiders/request.py b/scrapling/spiders/request.py
index ef766b2..8809c15 100644
--- a/scrapling/spiders/request.py
+++ b/scrapling/spiders/request.py
@@ -4,6 +4,7 @@ from functools import cached_property
from urllib.parse import urlparse, urlencode
import orjson
+from curl_cffi.requests.utils import update_url_params
from w3lib.url import canonicalize_url
from scrapling.engines.toolbelt.custom import Response
@@ -96,11 +97,14 @@ class Request:
post_data = self._session_kwargs.get("json", {})
body = orjson.dumps(post_data) if post_data else b""
+ params = self._session_kwargs.get("params")
+ url = update_url_params(self.url, params) if params else self.url
+
data: Dict[str, str | Tuple] = {
"sid": self.sid,
"body": body.hex(),
"method": self._session_kwargs.get("method", "GET"),
- "url": canonicalize_url(self.url, keep_fragments=keep_fragments),
+ "url": canonicalize_url(url, keep_fragments=keep_fragments),
}
if include_kwargs:
diff --git a/tests/spiders/test_request.py b/tests/spiders/test_request.py
index 00f6c49..296b660 100644
--- a/tests/spiders/test_request.py
+++ b/tests/spiders/test_request.py
@@ -131,6 +131,34 @@ class TestRequestProperties:
assert r1.update_fingerprint(include_headers=True) != r2.update_fingerprint(include_headers=True)
+ def test_fingerprint_includes_params(self):
+ r1 = Request("https://example.com/p?a=1", params={"skip": 1})
+ r2 = Request("https://example.com/p?a=1", params={"skip": 2})
+ r3 = Request("https://example.com/p?skip=2&a=1")
+
+ assert r1.update_fingerprint() != r2.update_fingerprint()
+ assert r2.update_fingerprint() == r3.update_fingerprint()
+
+ def test_fingerprint_params_override_existing_query_key(self):
+ r1 = Request("https://example.com/p?skip=0", params={"skip": 1})
+ r2 = Request("https://example.com/p?skip=1")
+
+ assert r1.update_fingerprint() == r2.update_fingerprint()
+
+ def test_fingerprint_params_keep_fragment(self):
+ r1 = Request("https://example.com/p#top", params={"skip": 1})
+ r2 = Request("https://example.com/p?skip=1#top")
+ r3 = Request("https://example.com/p#top")
+
+ assert r1.update_fingerprint(keep_fragments=True) == r2.update_fingerprint(keep_fragments=True)
+ assert r1.update_fingerprint() != r3.update_fingerprint()
+
+ def test_fingerprint_params_bool_and_list_values(self):
+ r1 = Request("https://example.com/p", params={"flag": True, "tag": ["x", "y"]})
+ r2 = Request("https://example.com/p?flag=true&tag=x&tag=y")
+
+ assert r1.update_fingerprint() == r2.update_fingerprint()
+
class TestRequestCopy:
"""Test Request copy functionality."""