Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Preserve leading $ in curl request data (#8292)

karanbayasa0951524990e
diff --git a/scrapy/utils/curl.py b/scrapy/utils/curl.py
index 2e0f5e23c..bc98fbec3 100644
--- a/scrapy/utils/curl.py
+++ b/scrapy/utils/curl.py
@@ -22,7 +22,6 @@ class DataAction(argparse.Action):
option_string: str | None = None,
) -> None:
value = str(values)
- value = value.removeprefix("$")
# curl merges repeated -d/--data/--data-raw options into a single body
# joined with "&"; mirror that instead of keeping only the last one.
previous = getattr(namespace, self.dest, None)
@@ -95,6 +94,47 @@ def _parse_headers_and_cookies(
return headers, cookies
+def _normalize_curl_command(curl_command: str) -> str:
+ """Normalize Bash ANSI-C quote prefixes for shlex."""
+ result: list[str] = []
+ in_single_quote = False
+ in_double_quote = False
+ escaped = False
+
+ i = 0
+ while i < len(curl_command):
+ char = curl_command[i]
+
+ if escaped:
+ result.append(char)
+ escaped = False
+ elif char == "\\" and not in_single_quote:
+ result.append(char)
+ escaped = True
+ elif char == "'" and not in_double_quote:
+ in_single_quote = not in_single_quote
+ result.append(char)
+ elif char == '"' and not in_single_quote:
+ in_double_quote = not in_double_quote
+ result.append(char)
+ elif (
+ char == "$"
+ and i + 1 < len(curl_command)
+ and curl_command[i + 1] == "'"
+ and not in_single_quote
+ and not in_double_quote
+ ):
+ result.append("'")
+ in_single_quote = True
+ i += 1
+ else:
+ result.append(char)
+
+ i += 1
+
+ return "".join(result)
+
+
def curl_to_request_kwargs(
curl_command: str, ignore_unknown_options: bool = True
) -> dict[str, Any]:
@@ -107,7 +147,7 @@ def curl_to_request_kwargs(
:return: dictionary of Request kwargs
"""
- curl_args = split(curl_command)
+ curl_args = split(_normalize_curl_command(curl_command))
if not curl_args or curl_args[0] != "curl":
raise ValueError('A curl command must start with "curl"')
diff --git a/tests/test_utils_curl.py b/tests/test_utils_curl.py
index 53ebf88a3..38c1c3f5a 100644
--- a/tests/test_utils_curl.py
+++ b/tests/test_utils_curl.py
@@ -25,6 +25,14 @@ class TestCurlToRequestKwargs:
expected_result = {"method": "GET", "url": "http://example.org/"}
self._test_command(curl_command, expected_result)
+ def test_get_url_with_escaped_space(self):
+ curl_command = r"curl http://example.org/a\ b"
+ expected_result = {
+ "method": "GET",
+ "url": "http://example.org/a b",
+ }
+ self._test_command(curl_command, expected_result)
+
def test_get_without_scheme(self):
curl_command = "curl www.example.org"
expected_result = {"method": "GET", "url": "http://www.example.org"}
@@ -60,6 +68,15 @@ class TestCurlToRequestKwargs:
}
self._test_command(curl_command, expected_result)
+ def test_get_header_with_string_prefix(self):
+ curl_command = "curl 'http://example.org/' -H $'X-A: b'"
+ expected_result = {
+ "method": "GET",
+ "url": "http://example.org/",
+ "headers": [("X-A", "b")],
+ }
+ self._test_command(curl_command, expected_result)
+
@pytest.mark.parametrize("header", ["X-Flag:", "X-Flag: "])
def test_get_header_without_value(self, header):
curl_command = f'curl "http://example.org/" -H "{header}"'
@@ -237,6 +254,15 @@ class TestCurlToRequestKwargs:
}
self._test_command(curl_command, expected_result)
+ def test_post_data_raw_preserves_leading_dollar(self):
+ curl_command = "curl 'https://www.example.org/' --data-raw '$top=10&$skip=5'"
+ expected_result = {
+ "method": "POST",
+ "url": "https://www.example.org/",
+ "body": "$top=10&$skip=5",
+ }
+ self._test_command(curl_command, expected_result)
+
def test_post_data_multiple(self):
# curl merges repeated -d/--data/--data-raw options into a single body
# joined with "&"; scrapy must do the same, not keep only the last one.
@@ -260,8 +286,8 @@ class TestCurlToRequestKwargs:
self._test_command(curl_command, expected_result)
def test_post_data_multiple_with_string_prefix(self):
- # The leading "$" left by bash $'...' quoting is stripped per option,
- # before the values are merged.
+ # Bash $'...' quoting is normalized before parsing,
+ # so the prefix is not included when the values are merged.
curl_command = "curl 'https://www.example.org/' -d $'a=1' -d $'b=2'"
expected_result = {
"method": "POST",