Scrapy, a fast high-level web crawling & scraping framework for Python.
Preserve leading $ in curl request data (#8292)
karanbayasa0951524990e
diff --git a/scrapy/utils/curl.py b/scrapy/utils/curl.pyindex 2e0f5e23c..bc98fbec3 100644--- a/scrapy/utils/curl.py+++ b/scrapy/utils/curl.py@@ -22,7 +22,6 @@ class DataAction(argparse.Action):option_string: str | None = None,) -> None:value = str(values)- value = value.removeprefix("$")# curl merges repeated -d/--data/--data-raw options into a single body# joined with "&"; mirror that instead of keeping only the last one.previous = getattr(namespace, self.dest, None)@@ -95,6 +94,47 @@ def _parse_headers_and_cookies(return headers, cookies+def _normalize_curl_command(curl_command: str) -> str:+ """Normalize Bash ANSI-C quote prefixes for shlex."""+ result: list[str] = []+ in_single_quote = False+ in_double_quote = False+ escaped = False++ i = 0+ while i < len(curl_command):+ char = curl_command[i]++ if escaped:+ result.append(char)+ escaped = False+ elif char == "\\" and not in_single_quote:+ result.append(char)+ escaped = True+ elif char == "'" and not in_double_quote:+ in_single_quote = not in_single_quote+ result.append(char)+ elif char == '"' and not in_single_quote:+ in_double_quote = not in_double_quote+ result.append(char)+ elif (+ char == "$"+ and i + 1 < len(curl_command)+ and curl_command[i + 1] == "'"+ and not in_single_quote+ and not in_double_quote+ ):+ result.append("'")+ in_single_quote = True+ i += 1+ else:+ result.append(char)++ i += 1++ return "".join(result)++def curl_to_request_kwargs(curl_command: str, ignore_unknown_options: bool = True) -> dict[str, Any]:@@ -107,7 +147,7 @@ def curl_to_request_kwargs(:return: dictionary of Request kwargs"""- curl_args = split(curl_command)+ curl_args = split(_normalize_curl_command(curl_command))if not curl_args or curl_args[0] != "curl":raise ValueError('A curl command must start with "curl"')diff --git a/tests/test_utils_curl.py b/tests/test_utils_curl.pyindex 53ebf88a3..38c1c3f5a 100644--- a/tests/test_utils_curl.py+++ b/tests/test_utils_curl.py@@ -25,6 +25,14 @@ class TestCurlToRequestKwargs:expected_result = {"method": "GET", "url": "http://example.org/"}self._test_command(curl_command, expected_result)+ def test_get_url_with_escaped_space(self):+ curl_command = r"curl http://example.org/a\ b"+ expected_result = {+ "method": "GET",+ "url": "http://example.org/a b",+ }+ self._test_command(curl_command, expected_result)+def test_get_without_scheme(self):curl_command = "curl www.example.org"expected_result = {"method": "GET", "url": "http://www.example.org"}@@ -60,6 +68,15 @@ class TestCurlToRequestKwargs:}self._test_command(curl_command, expected_result)+ def test_get_header_with_string_prefix(self):+ curl_command = "curl 'http://example.org/' -H $'X-A: b'"+ expected_result = {+ "method": "GET",+ "url": "http://example.org/",+ "headers": [("X-A", "b")],+ }+ self._test_command(curl_command, expected_result)+@pytest.mark.parametrize("header", ["X-Flag:", "X-Flag: "])def test_get_header_without_value(self, header):curl_command = f'curl "http://example.org/" -H "{header}"'@@ -237,6 +254,15 @@ class TestCurlToRequestKwargs:}self._test_command(curl_command, expected_result)+ def test_post_data_raw_preserves_leading_dollar(self):+ curl_command = "curl 'https://www.example.org/' --data-raw '$top=10&$skip=5'"+ expected_result = {+ "method": "POST",+ "url": "https://www.example.org/",+ "body": "$top=10&$skip=5",+ }+ self._test_command(curl_command, expected_result)+def test_post_data_multiple(self):# curl merges repeated -d/--data/--data-raw options into a single body# joined with "&"; scrapy must do the same, not keep only the last one.@@ -260,8 +286,8 @@ class TestCurlToRequestKwargs:self._test_command(curl_command, expected_result)def test_post_data_multiple_with_string_prefix(self):- # The leading "$" left by bash $'...' quoting is stripped per option,- # before the values are merged.+ # Bash $'...' quoting is normalized before parsing,+ # so the prefix is not included when the values are merged.curl_command = "curl 'https://www.example.org/' -d $'a=1' -d $'b=2'"expected_result = {"method": "POST",