Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Handle repeated, None and non-UTF-8 headers in curl integration. (#8267)

Andrey Rakhmatullin82e35561f1c4
diff --git a/scrapy/utils/curl.py b/scrapy/utils/curl.py
index 1591908d6..2e0f5e23c 100644
--- a/scrapy/utils/curl.py
+++ b/scrapy/utils/curl.py
@@ -61,20 +61,21 @@ for argument in safe_to_ignore_arguments:
def _parse_headers_and_cookies(
parsed_args: argparse.Namespace,
-) -> tuple[list[tuple[str, str | bytes]], dict[str, str]]:
- headers: list[tuple[str, str | bytes]] = []
+) -> tuple[list[tuple[str, str | bytes | None]], dict[str, str]]:
+ headers: list[tuple[str, str | bytes | None]] = []
cookies: dict[str, str] = {}
for header in parsed_args.headers or ():
+ val: str | None
if ":" in header:
name, val = header.split(":", 1)
+ val = val.strip() or None
elif header.endswith(";") and header[:-1].strip():
name, val = header[:-1], ""
else:
continue
name = name.strip()
- val = val.strip()
if name.title() == "Cookie":
- for name, morsel in SimpleCookie(val).items():
+ for name, morsel in SimpleCookie(val or "").items():
cookies[name] = morsel.value
else:
headers.append((name, val))
diff --git a/scrapy/utils/request.py b/scrapy/utils/request.py
index 1e6b5bfb9..2d804669b 100644
--- a/scrapy/utils/request.py
+++ b/scrapy/utils/request.py
@@ -236,9 +236,18 @@ def request_to_curl(request: Request) -> str:
data = f"--data-raw '{request.body.decode('utf-8')}'" if request.body else ""
- headers = " ".join(
- f"-H '{k.decode()}: {v[0].decode()}'" for k, v in request.headers.items()
- )
+ header_args: list[str] = []
+ for name, values in request.headers.items():
+ str_name = to_unicode(name, errors="replace")
+ if not values:
+ # "Remove an internal header by giving a replacement without
+ # content on the right side of the colon, as in: -H "Host:"."
+ # (which is how Scrapy handles empty headers)
+ header_args.append(f"-H '{str_name}:'")
+ for value in values:
+ str_value = to_unicode(value, errors="replace")
+ header_args.append(f"-H '{str_name}: {str_value}'")
+ headers = " ".join(header_args)
url = request.url
diff --git a/tests/test_utils_curl.py b/tests/test_utils_curl.py
index bd3442e07..53ebf88a3 100644
--- a/tests/test_utils_curl.py
+++ b/tests/test_utils_curl.py
@@ -60,6 +60,16 @@ class TestCurlToRequestKwargs:
}
self._test_command(curl_command, expected_result)
+ @pytest.mark.parametrize("header", ["X-Flag:", "X-Flag: "])
+ def test_get_header_without_value(self, header):
+ curl_command = f'curl "http://example.org/" -H "{header}"'
+ expected_result = {
+ "method": "GET",
+ "url": "http://example.org/",
+ "headers": [("X-Flag", None)],
+ }
+ self._test_command(curl_command, expected_result)
+
@pytest.mark.parametrize("header", ["X-Flag", "X-Flag;extra", "X-Flag ; ", ";"])
def test_get_header_curl_would_not_send(self, header):
curl_command = f'curl "http://example.org/" -H "{header}"'
diff --git a/tests/test_utils_request.py b/tests/test_utils_request.py
index 75c5fe1dc..32130ded8 100644
--- a/tests/test_utils_request.py
+++ b/tests/test_utils_request.py
@@ -407,6 +407,30 @@ class TestRequestToCurl:
)
self._test_request(request_object, expected_curl_command)
+ @pytest.mark.parametrize(
+ ("headers", "expected_args"),
+ [
+ ({"X-Multi": ["a", "b"]}, "-H 'X-Multi: a' -H 'X-Multi: b'"),
+ (
+ {"User-Agent": None, "Accept": "*/*"},
+ "-H 'User-Agent:' -H 'Accept: */*'",
+ ),
+ ({"X-Latin": b"caf\xe9"}, "-H 'X-Latin: caf\ufffd'"),
+ ],
+ )
+ def test_header_values(self, headers: dict[str, Any], expected_args: str) -> None:
+ request_object = Request("https://www.example.com", headers=headers)
+ expected_curl_command = f"curl -X GET https://www.example.com {expected_args}"
+ self._test_request(request_object, expected_curl_command)
+
+ def test_headers_from_curl(self) -> None:
+ request_object = Request(
+ "https://www.example.com",
+ headers={"User-Agent": None, "X-Multi": ["a", "b"]},
+ )
+ request_object2 = Request.from_curl(request_object.to_curl())
+ assert request_object2.headers == request_object.headers
+
def test_cookies_dict(self) -> None:
request_object = Request(
"https://www.httpbin.org/post",