Scrapy, a fast high-level web crawling & scraping framework for Python.
Handle repeated, None and non-UTF-8 headers in curl integration. (#8267)
Andrey Rakhmatullin82e35561f1c4
diff --git a/scrapy/utils/curl.py b/scrapy/utils/curl.pyindex 1591908d6..2e0f5e23c 100644--- a/scrapy/utils/curl.py+++ b/scrapy/utils/curl.py@@ -61,20 +61,21 @@ for argument in safe_to_ignore_arguments:def _parse_headers_and_cookies(parsed_args: argparse.Namespace,-) -> tuple[list[tuple[str, str | bytes]], dict[str, str]]:- headers: list[tuple[str, str | bytes]] = []+) -> tuple[list[tuple[str, str | bytes | None]], dict[str, str]]:+ headers: list[tuple[str, str | bytes | None]] = []cookies: dict[str, str] = {}for header in parsed_args.headers or ():+ val: str | Noneif ":" in header:name, val = header.split(":", 1)+ val = val.strip() or Noneelif header.endswith(";") and header[:-1].strip():name, val = header[:-1], ""else:continuename = name.strip()- val = val.strip()if name.title() == "Cookie":- for name, morsel in SimpleCookie(val).items():+ for name, morsel in SimpleCookie(val or "").items():cookies[name] = morsel.valueelse:headers.append((name, val))diff --git a/scrapy/utils/request.py b/scrapy/utils/request.pyindex 1e6b5bfb9..2d804669b 100644--- a/scrapy/utils/request.py+++ b/scrapy/utils/request.py@@ -236,9 +236,18 @@ def request_to_curl(request: Request) -> str:data = f"--data-raw '{request.body.decode('utf-8')}'" if request.body else ""- headers = " ".join(- f"-H '{k.decode()}: {v[0].decode()}'" for k, v in request.headers.items()- )+ header_args: list[str] = []+ for name, values in request.headers.items():+ str_name = to_unicode(name, errors="replace")+ if not values:+ # "Remove an internal header by giving a replacement without+ # content on the right side of the colon, as in: -H "Host:"."+ # (which is how Scrapy handles empty headers)+ header_args.append(f"-H '{str_name}:'")+ for value in values:+ str_value = to_unicode(value, errors="replace")+ header_args.append(f"-H '{str_name}: {str_value}'")+ headers = " ".join(header_args)url = request.urldiff --git a/tests/test_utils_curl.py b/tests/test_utils_curl.pyindex bd3442e07..53ebf88a3 100644--- a/tests/test_utils_curl.py+++ b/tests/test_utils_curl.py@@ -60,6 +60,16 @@ class TestCurlToRequestKwargs:}self._test_command(curl_command, expected_result)+ @pytest.mark.parametrize("header", ["X-Flag:", "X-Flag: "])+ def test_get_header_without_value(self, header):+ curl_command = f'curl "http://example.org/" -H "{header}"'+ expected_result = {+ "method": "GET",+ "url": "http://example.org/",+ "headers": [("X-Flag", None)],+ }+ self._test_command(curl_command, expected_result)+@pytest.mark.parametrize("header", ["X-Flag", "X-Flag;extra", "X-Flag ; ", ";"])def test_get_header_curl_would_not_send(self, header):curl_command = f'curl "http://example.org/" -H "{header}"'diff --git a/tests/test_utils_request.py b/tests/test_utils_request.pyindex 75c5fe1dc..32130ded8 100644--- a/tests/test_utils_request.py+++ b/tests/test_utils_request.py@@ -407,6 +407,30 @@ class TestRequestToCurl:)self._test_request(request_object, expected_curl_command)+ @pytest.mark.parametrize(+ ("headers", "expected_args"),+ [+ ({"X-Multi": ["a", "b"]}, "-H 'X-Multi: a' -H 'X-Multi: b'"),+ (+ {"User-Agent": None, "Accept": "*/*"},+ "-H 'User-Agent:' -H 'Accept: */*'",+ ),+ ({"X-Latin": b"caf\xe9"}, "-H 'X-Latin: caf\ufffd'"),+ ],+ )+ def test_header_values(self, headers: dict[str, Any], expected_args: str) -> None:+ request_object = Request("https://www.example.com", headers=headers)+ expected_curl_command = f"curl -X GET https://www.example.com {expected_args}"+ self._test_request(request_object, expected_curl_command)++ def test_headers_from_curl(self) -> None:+ request_object = Request(+ "https://www.example.com",+ headers={"User-Agent": None, "X-Multi": ["a", "b"]},+ )+ request_object2 = Request.from_curl(request_object.to_curl())+ assert request_object2.headers == request_object.headers+def test_cookies_dict(self) -> None:request_object = Request("https://www.httpbin.org/post",