🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev
Python aiai-scrapingautomationcrawlercrawlingcrawling-pythondatadata-extractionmcpmcp-serverplaywrightpythonscrapingselectorsstealthweb-scraperweb-scrapingweb-scraping-pythonwebscrapingxpath
fix(fetchers): preserve body precedence and None values in form encoding
Karim shoair1f02e3c89a4e
diff --git a/scrapling/engines/static.py b/scrapling/engines/static.pyindex 1829699..e701650 100644--- a/scrapling/engines/static.py+++ b/scrapling/engines/static.py@@ -157,13 +157,13 @@ class _ConfigurationLogic(ABC):if k not in skip_keys and v is not None:final_args[k] = v- # curl_cffi doesn't expand list values in a dict `data` and sends `None` as the text "None"- if isinstance(final_args.get("params"), dict):- final_args["params"] = {k: v for k, v in final_args["params"].items() if v is not None}- # With `multipart`, curl_cffi adds the items of a dict `data` as form parts, so it has to stay a dict- if isinstance(final_args.get("data"), dict) and not final_args.get("multipart"):- form = {k: v for k, v in final_args["data"].items() if v is not None}- final_args["data"] = urlencode(form, doseq=True)+ if (+ isinstance(final_args.get("data"), dict)+ and final_args.get("json") is None+ and final_args.get("content") is None+ and not final_args.get("multipart")+ ):+ final_args["data"] = urlencode(final_args["data"], doseq=True)if not any(k.lower() == "content-type" for k in final_args["headers"]):final_args["headers"]["Content-Type"] = "application/x-www-form-urlencoded"diff --git a/tests/fetchers/sync/test_requests.py b/tests/fetchers/sync/test_requests.pyindex a525b88..3f0b0fe 100644--- a/tests/fetchers/sync/test_requests.py+++ b/tests/fetchers/sync/test_requests.py@@ -163,10 +163,9 @@ class TestFetcher:body = fetcher.post(self.post_url, data={"tags": ["a", "b"], "ids": [1, 2]}).json()assert body["form"] == {"tags": ["a", "b"], "ids": ["1", "2"]}- def test_post_dict_data_none_values_are_dropped(self, fetcher):- """A ``None`` value in a dict ``data`` leaves the key out of the body."""+ def test_post_dict_data_none_values_are_preserved(self, fetcher):body = fetcher.post(self.post_url, data={"opt": None, "q": "x"}).json()- assert body["form"] == {"q": "x"}+ assert body["form"] == {"opt": "None", "q": "x"}def test_post_dict_data_scalars_and_form_content_type(self, fetcher):body = fetcher.post(self.post_url, data={"a": "x y", "b": 2}).json()@@ -177,10 +176,9 @@ class TestFetcher:body = fetcher.post(self.post_url, data="a=1&a=2").json()assert body["data"] == "a=1&a=2"- def test_get_params_none_values_are_dropped(self, fetcher):- """A ``None`` value in ``params`` leaves the key out of the query string."""+ def test_get_params_none_values_are_preserved(self, fetcher):body = fetcher.get(self.basic_url, params={"page": None, "q": "x"}).json()- assert body["args"] == {"q": "x"}+ assert body["args"] == {"page": "None", "q": "x"}def test_get_params_list_values_are_repeated_keys(self, fetcher):body = fetcher.get(self.basic_url, params={"tag": ["a", "b"]}).json()@@ -189,4 +187,23 @@ class TestFetcher:def test_session_post_dict_data_matches_fetcher(self):with FetcherSession() as session:body = session.post(self.post_url, data={"ids": [1, 2], "opt": None}).json()- assert body["form"] == {"ids": ["1", "2"]}+ assert body["form"] == {"ids": ["1", "2"], "opt": "None"}++ @pytest.mark.parametrize("method", ["post", "put"])+ @pytest.mark.parametrize("data", [{}, {"tags": ["a", "b"]}])+ @pytest.mark.parametrize("payload", [{"key": "value"}, {}, [], False, 0, ""])+ def test_json_with_data_keeps_json_body_and_content_type(self, fetcher, method, data, payload):+ body = getattr(fetcher, method)(getattr(self, f"{method}_url"), data=data, json=payload).json()+ assert body["headers"]["Content-Type"] == "application/json"+ assert body["json"] == payload+ assert body["form"] == {}++ @pytest.mark.parametrize("method", ["post", "put"])+ @pytest.mark.parametrize("data", [{}, {"tags": ["a", "b"]}])+ @pytest.mark.parametrize("content", [b"raw", b""])+ def test_content_with_data_keeps_raw_body(self, fetcher, method, data, content):+ body = getattr(fetcher, method)(getattr(self, f"{method}_url"), data=data, content=content).json()+ assert body["data"] == content.decode()+ assert body["form"] == {}+ if content:+ assert body["headers"]["Content-Type"] == "application/octet-stream"diff --git a/tests/fetchers/test_merge_request_args.py b/tests/fetchers/test_merge_request_args.pyindex a3b7cc8..fab0730 100644--- a/tests/fetchers/test_merge_request_args.py+++ b/tests/fetchers/test_merge_request_args.py@@ -99,9 +99,7 @@ class TestMergeRequestArgsSkipsBrowserParams:class TestMergeRequestArgsEncodesDataAndParams:- """Verify that dict `data` and `params` reach curl_cffi in the form the- `requests` convention expects: lists become repeated keys and None values- are dropped."""+ """Verify repeated form values and preserve existing query values."""def _build_args(self, **extra_kwargs):client = FetcherClient()@@ -119,9 +117,9 @@ class TestMergeRequestArgsEncodesDataAndParams:args = self._build_args(data={"t": ("a", "b"), "s": {"x"}})assert args["data"] == "t=a&t=b&s=x"- def test_data_none_value_is_dropped(self):+ def test_data_none_value_is_preserved(self):args = self._build_args(data={"opt": None, "q": "x"})- assert args["data"] == "q=x"+ assert args["data"] == "opt=None&q=x"def test_data_scalars_are_unchanged(self):args = self._build_args(data={"a": "1", "b": 2, "c": "x y"})@@ -136,9 +134,9 @@ class TestMergeRequestArgsEncodesDataAndParams:assert args["headers"]["content-type"] == "text/plain"assert "Content-Type" not in args["headers"]- def test_params_none_value_is_dropped(self):+ def test_params_none_value_is_preserved(self):args = self._build_args(params={"page": None, "q": "x"})- assert args["params"] == {"q": "x"}+ assert args["params"] == {"page": None, "q": "x"}def test_params_list_is_unchanged(self):args = self._build_args(params={"tag": ["a", "b"]})