Skip to content

🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev

fix(spiders): preserve falsy JSON bodies in request fingerprints (#480)

Alex7831c19eb32e
diff --git a/scrapling/spiders/request.py b/scrapling/spiders/request.py
index 0f87546..82e1d68 100644
--- a/scrapling/spiders/request.py
+++ b/scrapling/spiders/request.py
@@ -94,8 +94,8 @@ class Request:
elif isinstance(post_data, bytes):
body = post_data
else:
- post_data = self._session_kwargs.get("json", {})
- body = orjson.dumps(post_data) if post_data else b""
+ post_data = self._session_kwargs.get("json")
+ body = orjson.dumps(post_data) if post_data is not None else b""
params = self._session_kwargs.get("params")
url = update_url_params(self.url, params) if params else self.url
diff --git a/tests/spiders/test_request.py b/tests/spiders/test_request.py
index df580b5..2e92e95 100644
--- a/tests/spiders/test_request.py
+++ b/tests/spiders/test_request.py
@@ -100,6 +100,20 @@ class TestRequestProperties:
r2 = Request("https://example.com/page2")
assert r1.update_fingerprint() != r2.update_fingerprint()
+ @pytest.mark.parametrize("payload", [{}, [], False, 0, ""])
+ def test_fingerprint_preserves_falsy_json_bodies(self, payload):
+ url = "https://example.com/api"
+ json_request = Request(url, method="POST", json=payload)
+ empty_request = Request(url, method="POST")
+ assert json_request.update_fingerprint() != empty_request.update_fingerprint()
+
+ def test_fingerprint_distinguishes_empty_json_values(self):
+ fingerprints = {
+ Request("https://example.com/api", method="POST", json=payload).update_fingerprint()
+ for payload in ({}, [], False, 0, "")
+ }
+ assert len(fingerprints) == 5
+
def test_fingerprint_include_kwargs_uses_kwarg_values(self):
"""Test kwargs with different values produce different fingerprints."""
r1 = Request("https://example.com", timeout=1)