Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Decode Bash ANSI-C escapes in Request.from_curl() (#8433)

Hygor luz2aefa609f1a9
diff --git a/scrapy/utils/curl.py b/scrapy/utils/curl.py
index bc98fbec3..f040134d3 100644
--- a/scrapy/utils/curl.py
+++ b/scrapy/utils/curl.py
@@ -1,9 +1,10 @@
from __future__ import annotations
import argparse
+import re
import warnings
from http.cookies import SimpleCookie
-from shlex import split
+from shlex import quote, split
from typing import TYPE_CHECKING, Any, NoReturn
from urllib.parse import urlparse
@@ -94,8 +95,63 @@ def _parse_headers_and_cookies(
return headers, cookies
+_ANSI_C_QUOTED_RE = re.compile(r"\$'((?:[^'\\]|\\.)*)'", re.DOTALL)
+# https://www.gnu.org/software/bash/manual/html_node/ANSI_002dC-Quoting.html
+_ANSI_C_ESCAPE_RE = re.compile(
+ r"\\(?:([abeEfnrtv\\'\"?])|u([0-9A-Fa-f]{1,4})|U([0-9A-Fa-f]{1,8})|c(\\\\?|.))"
+ # A run of octal and hex escapes, each of them a byte.
+ r"|((?:\\(?:[0-7]{1,3}|x[0-9A-Fa-f]{1,2}))+)",
+ re.DOTALL,
+)
+_ANSI_C_SIMPLE_ESCAPES = {
+ "a": "\a",
+ "b": "\b",
+ "e": "\x1b",
+ "E": "\x1b",
+ "f": "\f",
+ "n": "\n",
+ "r": "\r",
+ "t": "\t",
+ "v": "\v",
+}
+# Maps the surrogates that the surrogateescape error handler uses for
+# undecodable bytes to the Latin-1 characters of those bytes.
+_SURROGATE_ESCAPES_TO_LATIN1 = {0xDC00 + byte: byte for byte in range(0x80, 0x100)}
+
+
+def _decode_ansi_c_escape(match: re.Match[str]) -> str:
+ simple, short_unicode, long_unicode, control, byte_escapes = match.groups()
+ if simple is not None:
+ return _ANSI_C_SIMPLE_ESCAPES.get(simple, simple)
+ if control is not None:
+ # \c\\ counts as \c followed by a single backslash.
+ return "\x7f" if control == "?" else chr(ord(control[0]) & 0x1F)
+ if byte_escapes is not None:
+ data = bytes(
+ int(escape[1:], 16) if escape[0] == "x" else int(escape, 8) & 0xFF
+ for escape in byte_escapes.split("\\")[1:]
+ )
+ # Bash (printf %q) writes non-ASCII characters as their UTF-8 bytes,
+ # while Firefox writes characters below U+0100 as \xHH, so bytes that
+ # are not valid UTF-8 are read as Latin-1.
+ return data.decode("utf-8", "surrogateescape").translate(
+ _SURROGATE_ESCAPES_TO_LATIN1
+ )
+ return chr(int(short_unicode or long_unicode, 16))
+
+
+def _decode_ansi_c_string(value: str) -> str:
+ """Decode *value*, the contents of a Bash ``$'...'`` string."""
+ decoded = _ANSI_C_ESCAPE_RE.sub(_decode_ansi_c_escape, value)
+ # Bash cuts the string at the first NUL byte.
+ decoded = decoded.partition("\0")[0]
+ # Firefox writes characters above U+FFFF as a UTF-16 surrogate pair of
+ # \u escapes; join them, and fail on unpaired surrogates.
+ return decoded.encode("utf-16-le", "surrogatepass").decode("utf-16-le")
+
+
def _normalize_curl_command(curl_command: str) -> str:
- """Normalize Bash ANSI-C quote prefixes for shlex."""
+ """Replace Bash ANSI-C quoted strings with their value quoted for shlex."""
result: list[str] = []
in_single_quote = False
in_double_quote = False
@@ -119,14 +175,13 @@ def _normalize_curl_command(curl_command: str) -> str:
result.append(char)
elif (
char == "$"
- and i + 1 < len(curl_command)
- and curl_command[i + 1] == "'"
and not in_single_quote
and not in_double_quote
+ and (match := _ANSI_C_QUOTED_RE.match(curl_command, i))
):
- result.append("'")
- in_single_quote = True
- i += 1
+ result.append(quote(_decode_ansi_c_string(match[1])))
+ i = match.end()
+ continue
else:
result.append(char)
diff --git a/tests/test_utils_curl.py b/tests/test_utils_curl.py
index 38c1c3f5a..17809adb8 100644
--- a/tests/test_utils_curl.py
+++ b/tests/test_utils_curl.py
@@ -296,6 +296,59 @@ class TestCurlToRequestKwargs:
}
self._test_command(curl_command, expected_result)
+ @pytest.mark.parametrize(
+ ("data", "body"),
+ [
+ (r"$'line1\nline2'", "line1\nline2"),
+ (r"""$'{"name":"O\'Brien"}'""", """{"name":"O'Brien"}"""),
+ (r"""$'\a\b\e\E\f\n\r\t\v\\\'\"\?'""", "\a\b\x1b\x1b\f\n\r\t\v\\'\"?"),
+ (r"$'\101\0101\1012'", "A\x081A2"),
+ (r"$'\303\251'", "é"),
+ (r"$'\x41\x4G\x414'", "A\x04GA4"),
+ (r"$'\u00e9\u20AC\u41G'", "é€AG"),
+ (r"$'\U0001F600\U41'", "\U0001f600A"),
+ (
+ r"""$'{"name":"O\'Brien","city":"Z\xfcrich \ud83d\ude00"}'""",
+ '{"name":"O\'Brien","city":"Zürich \U0001f600"}',
+ ),
+ (r"$'caf\xe9'", "café"),
+ (r"$'\303\251\xe9'", "éé"),
+ (r"$'\xff'", "ÿ"),
+ (r"$'\ca\cZ\c[\c?\c\\x'", "\x01\x1a\x1b\x7f\x1cx"),
+ (r"$'\z\q\8'", r"\z\q\8"),
+ (r"$'\x'", r"\x"),
+ (r"$'\u'", r"\u"),
+ (r"$'\c'", r"\c"),
+ (r"x$'a\0b'y", "xay"),
+ (r"$'A\400B'", "A"),
+ ("a$'b c'd", "ab cd"),
+ ("a$''b", "ab"),
+ (r"\$'a\nb'", r"$a\nb"),
+ (r'''"$'a\nb'"''', r"$'a\nb'"),
+ ],
+ )
+ def test_post_data_raw_ansi_c_quoting(self, data, body):
+ curl_command = f"curl 'https://www.example.org/' --data-raw {data}"
+ expected_result = {
+ "method": "POST",
+ "url": "https://www.example.org/",
+ "body": body,
+ }
+ self._test_command(curl_command, expected_result)
+
+ @pytest.mark.parametrize(
+ ("data", "match"),
+ [
+ ("$'abc", "No closing quotation"),
+ (r"$'\ud800'", "can't decode"),
+ (r"$'\ude00\ud83d'", "can't decode"),
+ (r"$'\U00110000'", r"chr\(\) arg not in range"),
+ ],
+ )
+ def test_post_data_raw_ansi_c_quoting_error(self, data, match):
+ with pytest.raises(ValueError, match=match):
+ curl_to_request_kwargs(f"curl 'https://www.example.org/' --data-raw {data}")
+
def test_explicit_get_with_data(self):
curl_command = "curl httpbin.org/anything -X GET --data asdf"
expected_result = {