Scrapy, a fast high-level web crawling & scraping framework for Python.
Handle valid curl inputs in Request.from_curl (#8131) (#8138)
Vinícius Silva Campos86bec33cd3e6
diff --git a/scrapy/utils/curl.py b/scrapy/utils/curl.pyindex c82440dc7..2651f43b8 100644--- a/scrapy/utils/curl.py+++ b/scrapy/utils/curl.py@@ -65,7 +65,12 @@ def _parse_headers_and_cookies(headers: list[tuple[str, bytes]] = []cookies: dict[str, str] = {}for header in parsed_args.headers or ():- name, val = header.split(":", 1)+ if ":" in header:+ name, val = header.split(":", 1)+ elif header.endswith(";") and header[:-1].strip():+ name, val = header[:-1], ""+ else:+ continuename = name.strip()val = val.strip()if name.title() == "Cookie":@@ -83,7 +88,7 @@ def _parse_headers_and_cookies(cookies[name] = morsel.valueif parsed_args.auth:- user, password = parsed_args.auth.split(":", 1)+ user, _, password = parsed_args.auth.partition(":")headers.append(("Authorization", basic_auth_header(user, password)))return headers, cookies@@ -103,7 +108,7 @@ def curl_to_request_kwargs(curl_args = split(curl_command)- if curl_args[0] != "curl":+ if not curl_args or curl_args[0] != "curl":raise ValueError('A curl command must start with "curl"')parsed_args, argv = curl_parser.parse_known_args(curl_args[1:])diff --git a/tests/test_utils_curl.py b/tests/test_utils_curl.pyindex 8ab6c5c2f..57ae5d9e1 100644--- a/tests/test_utils_curl.py+++ b/tests/test_utils_curl.py@@ -41,6 +41,40 @@ class TestCurlToRequestKwargs:}self._test_command(curl_command, expected_result)+ @pytest.mark.parametrize("auth", ["some_username", "some_username:"])+ def test_get_basic_auth_without_password(self, auth):+ curl_command = f'curl "https://api.test.com/" -u "{auth}"'+ expected_result = {+ "method": "GET",+ "url": "https://api.test.com/",+ "headers": [("Authorization", basic_auth_header("some_username", ""))],+ }+ self._test_command(curl_command, expected_result)++ def test_get_header_with_empty_value(self):+ curl_command = 'curl "http://example.org/" -H "X-Flag;"'+ expected_result = {+ "method": "GET",+ "url": "http://example.org/",+ "headers": [("X-Flag", "")],+ }+ self._test_command(curl_command, expected_result)++ @pytest.mark.parametrize("header", ["X-Flag", "X-Flag;extra", "X-Flag ; ", ";"])+ def test_get_header_curl_would_not_send(self, header):+ curl_command = f'curl "http://example.org/" -H "{header}"'+ expected_result = {"method": "GET", "url": "http://example.org/"}+ self._test_command(curl_command, expected_result)++ def test_get_ignored_header_does_not_hide_following_header(self):+ curl_command = 'curl "http://example.org/" -H "X-Flag" -H "X-Other: value"'+ expected_result = {+ "method": "GET",+ "url": "http://example.org/",+ "headers": [("X-Other", "value")],+ }+ self._test_command(curl_command, expected_result)+def test_get_cookie_option(self):curl_command = 'curl "http://example.org/" -b "a=1; b=2"'expected_result = {@@ -285,3 +319,7 @@ class TestCurlToRequestKwargs:def test_must_start_with_curl_error(self):with pytest.raises(ValueError, match="A curl command must start"):curl_to_request_kwargs("carl -X POST http://example.org")++ def test_empty_command_error(self):+ with pytest.raises(ValueError, match="A curl command must start"):+ curl_to_request_kwargs("")