🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev
Python aiai-scrapingautomationcrawlercrawlingcrawling-pythondatadata-extractionmcpmcp-serverplaywrightpythonscrapingselectorsstealthweb-scraperweb-scrapingweb-scraping-pythonwebscrapingxpath
fix(fetchers): encode list values and drop None values in dict data and params (#485)
Suliman Abdulrazzaq2dbeb48e38db
diff --git a/scrapling/engines/static.py b/scrapling/engines/static.pyindex 1c14d26..1829699 100644--- a/scrapling/engines/static.py+++ b/scrapling/engines/static.py@@ -1,6 +1,7 @@from abc import ABCfrom random import choicefrom time import sleep as time_sleep+from urllib.parse import urlencodefrom asyncio import sleep as asyncio_sleepfrom curl_cffi.curl import CurlError@@ -156,6 +157,16 @@ class _ConfigurationLogic(ABC):if k not in skip_keys and v is not None:final_args[k] = v+ # curl_cffi doesn't expand list values in a dict `data` and sends `None` as the text "None"+ if isinstance(final_args.get("params"), dict):+ final_args["params"] = {k: v for k, v in final_args["params"].items() if v is not None}+ # With `multipart`, curl_cffi adds the items of a dict `data` as form parts, so it has to stay a dict+ if isinstance(final_args.get("data"), dict) and not final_args.get("multipart"):+ form = {k: v for k, v in final_args["data"].items() if v is not None}+ final_args["data"] = urlencode(form, doseq=True)+ if not any(k.lower() == "content-type" for k in final_args["headers"]):+ final_args["headers"]["Content-Type"] = "application/x-www-form-urlencoded"+if http3_enabled: # pragma: no coverfinal_args["http_version"] = CurlHttpVersion.V3ONLYif impersonate:diff --git a/tests/fetchers/sync/test_requests.py b/tests/fetchers/sync/test_requests.pyindex 5f9c03b..a525b88 100644--- a/tests/fetchers/sync/test_requests.py+++ b/tests/fetchers/sync/test_requests.py@@ -2,6 +2,7 @@ import pytestimport pytest_httpbinfrom scrapling import Fetcher+from scrapling.fetchers import FetcherSessionFetcher.adaptive = True@@ -156,3 +157,36 @@ class TestFetcher:"""``retries`` below 1 means "send the request once", not "send nothing"."""assert fetcher.get(self.status_200, retries=0).status == 200assert fetcher.get(self.status_200, retries=-1).status == 200++ def test_post_dict_data_list_values_are_repeated_keys(self, fetcher):+ """List values in a dict ``data`` are sent as repeated keys, not as their repr."""+ body = fetcher.post(self.post_url, data={"tags": ["a", "b"], "ids": [1, 2]}).json()+ assert body["form"] == {"tags": ["a", "b"], "ids": ["1", "2"]}++ def test_post_dict_data_none_values_are_dropped(self, fetcher):+ """A ``None`` value in a dict ``data`` leaves the key out of the body."""+ body = fetcher.post(self.post_url, data={"opt": None, "q": "x"}).json()+ assert body["form"] == {"q": "x"}++ def test_post_dict_data_scalars_and_form_content_type(self, fetcher):+ body = fetcher.post(self.post_url, data={"a": "x y", "b": 2}).json()+ assert body["form"] == {"a": "x y", "b": "2"}+ assert body["headers"]["Content-Type"] == "application/x-www-form-urlencoded"++ def test_post_string_data_is_untouched(self, fetcher):+ body = fetcher.post(self.post_url, data="a=1&a=2").json()+ assert body["data"] == "a=1&a=2"++ def test_get_params_none_values_are_dropped(self, fetcher):+ """A ``None`` value in ``params`` leaves the key out of the query string."""+ body = fetcher.get(self.basic_url, params={"page": None, "q": "x"}).json()+ assert body["args"] == {"q": "x"}++ def test_get_params_list_values_are_repeated_keys(self, fetcher):+ body = fetcher.get(self.basic_url, params={"tag": ["a", "b"]}).json()+ assert body["args"] == {"tag": ["a", "b"]}++ def test_session_post_dict_data_matches_fetcher(self):+ with FetcherSession() as session:+ body = session.post(self.post_url, data={"ids": [1, 2], "opt": None}).json()+ assert body["form"] == {"ids": ["1", "2"]}diff --git a/tests/fetchers/test_merge_request_args.py b/tests/fetchers/test_merge_request_args.pyindex d9185c3..a3b7cc8 100644--- a/tests/fetchers/test_merge_request_args.py+++ b/tests/fetchers/test_merge_request_args.py@@ -96,3 +96,69 @@ class TestMergeRequestArgsSkipsBrowserParams:"""Arbitrary curl_cffi-compatible kwargs should survive."""args = self._build_args(cookies={"session": "abc"})assert args.get("cookies") == {"session": "abc"}+++class TestMergeRequestArgsEncodesDataAndParams:+ """Verify that dict `data` and `params` reach curl_cffi in the form the+ `requests` convention expects: lists become repeated keys and None values+ are dropped."""++ def _build_args(self, **extra_kwargs):+ client = FetcherClient()+ return client._merge_request_args(url="https://example.com", **extra_kwargs)++ def test_data_list_becomes_repeated_keys(self):+ args = self._build_args(data={"tags": ["a", "b"]})+ assert args["data"] == "tags=a&tags=b"++ def test_data_int_list_becomes_repeated_keys(self):+ args = self._build_args(data={"ids": [1, 2]})+ assert args["data"] == "ids=1&ids=2"++ def test_data_tuple_and_set_values_are_expanded(self):+ args = self._build_args(data={"t": ("a", "b"), "s": {"x"}})+ assert args["data"] == "t=a&t=b&s=x"++ def test_data_none_value_is_dropped(self):+ args = self._build_args(data={"opt": None, "q": "x"})+ assert args["data"] == "q=x"++ def test_data_scalars_are_unchanged(self):+ args = self._build_args(data={"a": "1", "b": 2, "c": "x y"})+ assert args["data"] == "a=1&b=2&c=x+y"++ def test_data_dict_keeps_form_content_type(self):+ args = self._build_args(data={"tags": ["a", "b"]})+ assert args["headers"]["Content-Type"] == "application/x-www-form-urlencoded"++ def test_data_dict_keeps_caller_content_type(self):+ args = self._build_args(data={"a": "1"}, headers={"content-type": "text/plain"})+ assert args["headers"]["content-type"] == "text/plain"+ assert "Content-Type" not in args["headers"]++ def test_params_none_value_is_dropped(self):+ args = self._build_args(params={"page": None, "q": "x"})+ assert args["params"] == {"q": "x"}++ def test_params_list_is_unchanged(self):+ args = self._build_args(params={"tag": ["a", "b"]})+ assert args["params"] == {"tag": ["a", "b"]}++ def test_non_dict_data_is_untouched(self):+ pairs = [("tags", ["a", "b"]), ("opt", None)]+ assert self._build_args(data="a=1&b=2")["data"] == "a=1&b=2"+ assert self._build_args(data=b"raw")["data"] == b"raw"+ assert self._build_args(data=pairs)["data"] is pairs+ assert "Content-Type" not in self._build_args(data="a=1")["headers"]++ def test_non_dict_params_is_untouched(self):+ pairs = [("page", None), ("q", "x")]+ assert self._build_args(params=pairs)["params"] is pairs++ def test_dict_data_with_multipart_stays_a_dict(self):+ """curl_cffi adds the items of a dict `data` to a `multipart` body, so it must not become a string."""+ mime = object()+ args = self._build_args(data={"a": "1"}, multipart=mime)+ assert args["data"] == {"a": "1"}+ assert args["multipart"] is mime+ assert "Content-Type" not in args["headers"]