Skip to content

🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev

fix(fetchers): encode list values and drop None values in dict data and params (#485)

Suliman Abdulrazzaq2dbeb48e38db
diff --git a/scrapling/engines/static.py b/scrapling/engines/static.py
index 1c14d26..1829699 100644
--- a/scrapling/engines/static.py
+++ b/scrapling/engines/static.py
@@ -1,6 +1,7 @@
from abc import ABC
from random import choice
from time import sleep as time_sleep
+from urllib.parse import urlencode
from asyncio import sleep as asyncio_sleep
from curl_cffi.curl import CurlError
@@ -156,6 +157,16 @@ class _ConfigurationLogic(ABC):
if k not in skip_keys and v is not None:
final_args[k] = v
+ # curl_cffi doesn't expand list values in a dict `data` and sends `None` as the text "None"
+ if isinstance(final_args.get("params"), dict):
+ final_args["params"] = {k: v for k, v in final_args["params"].items() if v is not None}
+ # With `multipart`, curl_cffi adds the items of a dict `data` as form parts, so it has to stay a dict
+ if isinstance(final_args.get("data"), dict) and not final_args.get("multipart"):
+ form = {k: v for k, v in final_args["data"].items() if v is not None}
+ final_args["data"] = urlencode(form, doseq=True)
+ if not any(k.lower() == "content-type" for k in final_args["headers"]):
+ final_args["headers"]["Content-Type"] = "application/x-www-form-urlencoded"
+
if http3_enabled: # pragma: no cover
final_args["http_version"] = CurlHttpVersion.V3ONLY
if impersonate:
diff --git a/tests/fetchers/sync/test_requests.py b/tests/fetchers/sync/test_requests.py
index 5f9c03b..a525b88 100644
--- a/tests/fetchers/sync/test_requests.py
+++ b/tests/fetchers/sync/test_requests.py
@@ -2,6 +2,7 @@ import pytest
import pytest_httpbin
from scrapling import Fetcher
+from scrapling.fetchers import FetcherSession
Fetcher.adaptive = True
@@ -156,3 +157,36 @@ class TestFetcher:
"""``retries`` below 1 means "send the request once", not "send nothing"."""
assert fetcher.get(self.status_200, retries=0).status == 200
assert fetcher.get(self.status_200, retries=-1).status == 200
+
+ def test_post_dict_data_list_values_are_repeated_keys(self, fetcher):
+ """List values in a dict ``data`` are sent as repeated keys, not as their repr."""
+ body = fetcher.post(self.post_url, data={"tags": ["a", "b"], "ids": [1, 2]}).json()
+ assert body["form"] == {"tags": ["a", "b"], "ids": ["1", "2"]}
+
+ def test_post_dict_data_none_values_are_dropped(self, fetcher):
+ """A ``None`` value in a dict ``data`` leaves the key out of the body."""
+ body = fetcher.post(self.post_url, data={"opt": None, "q": "x"}).json()
+ assert body["form"] == {"q": "x"}
+
+ def test_post_dict_data_scalars_and_form_content_type(self, fetcher):
+ body = fetcher.post(self.post_url, data={"a": "x y", "b": 2}).json()
+ assert body["form"] == {"a": "x y", "b": "2"}
+ assert body["headers"]["Content-Type"] == "application/x-www-form-urlencoded"
+
+ def test_post_string_data_is_untouched(self, fetcher):
+ body = fetcher.post(self.post_url, data="a=1&a=2").json()
+ assert body["data"] == "a=1&a=2"
+
+ def test_get_params_none_values_are_dropped(self, fetcher):
+ """A ``None`` value in ``params`` leaves the key out of the query string."""
+ body = fetcher.get(self.basic_url, params={"page": None, "q": "x"}).json()
+ assert body["args"] == {"q": "x"}
+
+ def test_get_params_list_values_are_repeated_keys(self, fetcher):
+ body = fetcher.get(self.basic_url, params={"tag": ["a", "b"]}).json()
+ assert body["args"] == {"tag": ["a", "b"]}
+
+ def test_session_post_dict_data_matches_fetcher(self):
+ with FetcherSession() as session:
+ body = session.post(self.post_url, data={"ids": [1, 2], "opt": None}).json()
+ assert body["form"] == {"ids": ["1", "2"]}
diff --git a/tests/fetchers/test_merge_request_args.py b/tests/fetchers/test_merge_request_args.py
index d9185c3..a3b7cc8 100644
--- a/tests/fetchers/test_merge_request_args.py
+++ b/tests/fetchers/test_merge_request_args.py
@@ -96,3 +96,69 @@ class TestMergeRequestArgsSkipsBrowserParams:
"""Arbitrary curl_cffi-compatible kwargs should survive."""
args = self._build_args(cookies={"session": "abc"})
assert args.get("cookies") == {"session": "abc"}
+
+
+class TestMergeRequestArgsEncodesDataAndParams:
+ """Verify that dict `data` and `params` reach curl_cffi in the form the
+ `requests` convention expects: lists become repeated keys and None values
+ are dropped."""
+
+ def _build_args(self, **extra_kwargs):
+ client = FetcherClient()
+ return client._merge_request_args(url="https://example.com", **extra_kwargs)
+
+ def test_data_list_becomes_repeated_keys(self):
+ args = self._build_args(data={"tags": ["a", "b"]})
+ assert args["data"] == "tags=a&tags=b"
+
+ def test_data_int_list_becomes_repeated_keys(self):
+ args = self._build_args(data={"ids": [1, 2]})
+ assert args["data"] == "ids=1&ids=2"
+
+ def test_data_tuple_and_set_values_are_expanded(self):
+ args = self._build_args(data={"t": ("a", "b"), "s": {"x"}})
+ assert args["data"] == "t=a&t=b&s=x"
+
+ def test_data_none_value_is_dropped(self):
+ args = self._build_args(data={"opt": None, "q": "x"})
+ assert args["data"] == "q=x"
+
+ def test_data_scalars_are_unchanged(self):
+ args = self._build_args(data={"a": "1", "b": 2, "c": "x y"})
+ assert args["data"] == "a=1&b=2&c=x+y"
+
+ def test_data_dict_keeps_form_content_type(self):
+ args = self._build_args(data={"tags": ["a", "b"]})
+ assert args["headers"]["Content-Type"] == "application/x-www-form-urlencoded"
+
+ def test_data_dict_keeps_caller_content_type(self):
+ args = self._build_args(data={"a": "1"}, headers={"content-type": "text/plain"})
+ assert args["headers"]["content-type"] == "text/plain"
+ assert "Content-Type" not in args["headers"]
+
+ def test_params_none_value_is_dropped(self):
+ args = self._build_args(params={"page": None, "q": "x"})
+ assert args["params"] == {"q": "x"}
+
+ def test_params_list_is_unchanged(self):
+ args = self._build_args(params={"tag": ["a", "b"]})
+ assert args["params"] == {"tag": ["a", "b"]}
+
+ def test_non_dict_data_is_untouched(self):
+ pairs = [("tags", ["a", "b"]), ("opt", None)]
+ assert self._build_args(data="a=1&b=2")["data"] == "a=1&b=2"
+ assert self._build_args(data=b"raw")["data"] == b"raw"
+ assert self._build_args(data=pairs)["data"] is pairs
+ assert "Content-Type" not in self._build_args(data="a=1")["headers"]
+
+ def test_non_dict_params_is_untouched(self):
+ pairs = [("page", None), ("q", "x")]
+ assert self._build_args(params=pairs)["params"] is pairs
+
+ def test_dict_data_with_multipart_stays_a_dict(self):
+ """curl_cffi adds the items of a dict `data` to a `multipart` body, so it must not become a string."""
+ mime = object()
+ args = self._build_args(data={"a": "1"}, multipart=mime)
+ assert args["data"] == {"a": "1"}
+ assert args["multipart"] is mime
+ assert "Content-Type" not in args["headers"]