🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev
Python aiai-scrapingautomationcrawlercrawlingcrawling-pythondatadata-extractionmcpmcp-serverplaywrightpythonscrapingselectorsstealthweb-scraperweb-scrapingweb-scraping-pythonwebscrapingxpath
fix(spiders): preserve falsy JSON bodies in request fingerprints (#480)
Alex7831c19eb32e
diff --git a/scrapling/spiders/request.py b/scrapling/spiders/request.pyindex 0f87546..82e1d68 100644--- a/scrapling/spiders/request.py+++ b/scrapling/spiders/request.py@@ -94,8 +94,8 @@ class Request:elif isinstance(post_data, bytes):body = post_dataelse:- post_data = self._session_kwargs.get("json", {})- body = orjson.dumps(post_data) if post_data else b""+ post_data = self._session_kwargs.get("json")+ body = orjson.dumps(post_data) if post_data is not None else b""params = self._session_kwargs.get("params")url = update_url_params(self.url, params) if params else self.urldiff --git a/tests/spiders/test_request.py b/tests/spiders/test_request.pyindex df580b5..2e92e95 100644--- a/tests/spiders/test_request.py+++ b/tests/spiders/test_request.py@@ -100,6 +100,20 @@ class TestRequestProperties:r2 = Request("https://example.com/page2")assert r1.update_fingerprint() != r2.update_fingerprint()+ @pytest.mark.parametrize("payload", [{}, [], False, 0, ""])+ def test_fingerprint_preserves_falsy_json_bodies(self, payload):+ url = "https://example.com/api"+ json_request = Request(url, method="POST", json=payload)+ empty_request = Request(url, method="POST")+ assert json_request.update_fingerprint() != empty_request.update_fingerprint()++ def test_fingerprint_distinguishes_empty_json_values(self):+ fingerprints = {+ Request("https://example.com/api", method="POST", json=payload).update_fingerprint()+ for payload in ({}, [], False, 0, "")+ }+ assert len(fingerprints) == 5+def test_fingerprint_include_kwargs_uses_kwarg_values(self):"""Test kwargs with different values produce different fingerprints."""r1 = Request("https://example.com", timeout=1)