Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Handle valid curl inputs in Request.from_curl (#8131) (#8138)

Vinícius Silva Campos86bec33cd3e6
diff --git a/scrapy/utils/curl.py b/scrapy/utils/curl.py
index c82440dc7..2651f43b8 100644
--- a/scrapy/utils/curl.py
+++ b/scrapy/utils/curl.py
@@ -65,7 +65,12 @@ def _parse_headers_and_cookies(
headers: list[tuple[str, bytes]] = []
cookies: dict[str, str] = {}
for header in parsed_args.headers or ():
- name, val = header.split(":", 1)
+ if ":" in header:
+ name, val = header.split(":", 1)
+ elif header.endswith(";") and header[:-1].strip():
+ name, val = header[:-1], ""
+ else:
+ continue
name = name.strip()
val = val.strip()
if name.title() == "Cookie":
@@ -83,7 +88,7 @@ def _parse_headers_and_cookies(
cookies[name] = morsel.value
if parsed_args.auth:
- user, password = parsed_args.auth.split(":", 1)
+ user, _, password = parsed_args.auth.partition(":")
headers.append(("Authorization", basic_auth_header(user, password)))
return headers, cookies
@@ -103,7 +108,7 @@ def curl_to_request_kwargs(
curl_args = split(curl_command)
- if curl_args[0] != "curl":
+ if not curl_args or curl_args[0] != "curl":
raise ValueError('A curl command must start with "curl"')
parsed_args, argv = curl_parser.parse_known_args(curl_args[1:])
diff --git a/tests/test_utils_curl.py b/tests/test_utils_curl.py
index 8ab6c5c2f..57ae5d9e1 100644
--- a/tests/test_utils_curl.py
+++ b/tests/test_utils_curl.py
@@ -41,6 +41,40 @@ class TestCurlToRequestKwargs:
}
self._test_command(curl_command, expected_result)
+ @pytest.mark.parametrize("auth", ["some_username", "some_username:"])
+ def test_get_basic_auth_without_password(self, auth):
+ curl_command = f'curl "https://api.test.com/" -u "{auth}"'
+ expected_result = {
+ "method": "GET",
+ "url": "https://api.test.com/",
+ "headers": [("Authorization", basic_auth_header("some_username", ""))],
+ }
+ self._test_command(curl_command, expected_result)
+
+ def test_get_header_with_empty_value(self):
+ curl_command = 'curl "http://example.org/" -H "X-Flag;"'
+ expected_result = {
+ "method": "GET",
+ "url": "http://example.org/",
+ "headers": [("X-Flag", "")],
+ }
+ self._test_command(curl_command, expected_result)
+
+ @pytest.mark.parametrize("header", ["X-Flag", "X-Flag;extra", "X-Flag ; ", ";"])
+ def test_get_header_curl_would_not_send(self, header):
+ curl_command = f'curl "http://example.org/" -H "{header}"'
+ expected_result = {"method": "GET", "url": "http://example.org/"}
+ self._test_command(curl_command, expected_result)
+
+ def test_get_ignored_header_does_not_hide_following_header(self):
+ curl_command = 'curl "http://example.org/" -H "X-Flag" -H "X-Other: value"'
+ expected_result = {
+ "method": "GET",
+ "url": "http://example.org/",
+ "headers": [("X-Other", "value")],
+ }
+ self._test_command(curl_command, expected_result)
+
def test_get_cookie_option(self):
curl_command = 'curl "http://example.org/" -b "a=1; b=2"'
expected_result = {
@@ -285,3 +319,7 @@ class TestCurlToRequestKwargs:
def test_must_start_with_curl_error(self):
with pytest.raises(ValueError, match="A curl command must start"):
curl_to_request_kwargs("carl -X POST http://example.org")
+
+ def test_empty_command_error(self):
+ with pytest.raises(ValueError, match="A curl command must start"):
+ curl_to_request_kwargs("")