Skip to content

🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev

fix(fetchers): preserve body precedence and None values in form encoding

Karim shoair1f02e3c89a4e
diff --git a/scrapling/engines/static.py b/scrapling/engines/static.py
index 1829699..e701650 100644
--- a/scrapling/engines/static.py
+++ b/scrapling/engines/static.py
@@ -157,13 +157,13 @@ class _ConfigurationLogic(ABC):
if k not in skip_keys and v is not None:
final_args[k] = v
- # curl_cffi doesn't expand list values in a dict `data` and sends `None` as the text "None"
- if isinstance(final_args.get("params"), dict):
- final_args["params"] = {k: v for k, v in final_args["params"].items() if v is not None}
- # With `multipart`, curl_cffi adds the items of a dict `data` as form parts, so it has to stay a dict
- if isinstance(final_args.get("data"), dict) and not final_args.get("multipart"):
- form = {k: v for k, v in final_args["data"].items() if v is not None}
- final_args["data"] = urlencode(form, doseq=True)
+ if (
+ isinstance(final_args.get("data"), dict)
+ and final_args.get("json") is None
+ and final_args.get("content") is None
+ and not final_args.get("multipart")
+ ):
+ final_args["data"] = urlencode(final_args["data"], doseq=True)
if not any(k.lower() == "content-type" for k in final_args["headers"]):
final_args["headers"]["Content-Type"] = "application/x-www-form-urlencoded"
diff --git a/tests/fetchers/sync/test_requests.py b/tests/fetchers/sync/test_requests.py
index a525b88..3f0b0fe 100644
--- a/tests/fetchers/sync/test_requests.py
+++ b/tests/fetchers/sync/test_requests.py
@@ -163,10 +163,9 @@ class TestFetcher:
body = fetcher.post(self.post_url, data={"tags": ["a", "b"], "ids": [1, 2]}).json()
assert body["form"] == {"tags": ["a", "b"], "ids": ["1", "2"]}
- def test_post_dict_data_none_values_are_dropped(self, fetcher):
- """A ``None`` value in a dict ``data`` leaves the key out of the body."""
+ def test_post_dict_data_none_values_are_preserved(self, fetcher):
body = fetcher.post(self.post_url, data={"opt": None, "q": "x"}).json()
- assert body["form"] == {"q": "x"}
+ assert body["form"] == {"opt": "None", "q": "x"}
def test_post_dict_data_scalars_and_form_content_type(self, fetcher):
body = fetcher.post(self.post_url, data={"a": "x y", "b": 2}).json()
@@ -177,10 +176,9 @@ class TestFetcher:
body = fetcher.post(self.post_url, data="a=1&a=2").json()
assert body["data"] == "a=1&a=2"
- def test_get_params_none_values_are_dropped(self, fetcher):
- """A ``None`` value in ``params`` leaves the key out of the query string."""
+ def test_get_params_none_values_are_preserved(self, fetcher):
body = fetcher.get(self.basic_url, params={"page": None, "q": "x"}).json()
- assert body["args"] == {"q": "x"}
+ assert body["args"] == {"page": "None", "q": "x"}
def test_get_params_list_values_are_repeated_keys(self, fetcher):
body = fetcher.get(self.basic_url, params={"tag": ["a", "b"]}).json()
@@ -189,4 +187,23 @@ class TestFetcher:
def test_session_post_dict_data_matches_fetcher(self):
with FetcherSession() as session:
body = session.post(self.post_url, data={"ids": [1, 2], "opt": None}).json()
- assert body["form"] == {"ids": ["1", "2"]}
+ assert body["form"] == {"ids": ["1", "2"], "opt": "None"}
+
+ @pytest.mark.parametrize("method", ["post", "put"])
+ @pytest.mark.parametrize("data", [{}, {"tags": ["a", "b"]}])
+ @pytest.mark.parametrize("payload", [{"key": "value"}, {}, [], False, 0, ""])
+ def test_json_with_data_keeps_json_body_and_content_type(self, fetcher, method, data, payload):
+ body = getattr(fetcher, method)(getattr(self, f"{method}_url"), data=data, json=payload).json()
+ assert body["headers"]["Content-Type"] == "application/json"
+ assert body["json"] == payload
+ assert body["form"] == {}
+
+ @pytest.mark.parametrize("method", ["post", "put"])
+ @pytest.mark.parametrize("data", [{}, {"tags": ["a", "b"]}])
+ @pytest.mark.parametrize("content", [b"raw", b""])
+ def test_content_with_data_keeps_raw_body(self, fetcher, method, data, content):
+ body = getattr(fetcher, method)(getattr(self, f"{method}_url"), data=data, content=content).json()
+ assert body["data"] == content.decode()
+ assert body["form"] == {}
+ if content:
+ assert body["headers"]["Content-Type"] == "application/octet-stream"
diff --git a/tests/fetchers/test_merge_request_args.py b/tests/fetchers/test_merge_request_args.py
index a3b7cc8..fab0730 100644
--- a/tests/fetchers/test_merge_request_args.py
+++ b/tests/fetchers/test_merge_request_args.py
@@ -99,9 +99,7 @@ class TestMergeRequestArgsSkipsBrowserParams:
class TestMergeRequestArgsEncodesDataAndParams:
- """Verify that dict `data` and `params` reach curl_cffi in the form the
- `requests` convention expects: lists become repeated keys and None values
- are dropped."""
+ """Verify repeated form values and preserve existing query values."""
def _build_args(self, **extra_kwargs):
client = FetcherClient()
@@ -119,9 +117,9 @@ class TestMergeRequestArgsEncodesDataAndParams:
args = self._build_args(data={"t": ("a", "b"), "s": {"x"}})
assert args["data"] == "t=a&t=b&s=x"
- def test_data_none_value_is_dropped(self):
+ def test_data_none_value_is_preserved(self):
args = self._build_args(data={"opt": None, "q": "x"})
- assert args["data"] == "q=x"
+ assert args["data"] == "opt=None&q=x"
def test_data_scalars_are_unchanged(self):
args = self._build_args(data={"a": "1", "b": 2, "c": "x y"})
@@ -136,9 +134,9 @@ class TestMergeRequestArgsEncodesDataAndParams:
assert args["headers"]["content-type"] == "text/plain"
assert "Content-Type" not in args["headers"]
- def test_params_none_value_is_dropped(self):
+ def test_params_none_value_is_preserved(self):
args = self._build_args(params={"page": None, "q": "x"})
- assert args["params"] == {"q": "x"}
+ assert args["params"] == {"page": None, "q": "x"}
def test_params_list_is_unchanged(self):
args = self._build_args(params={"tag": ["a", "b"]})