Scrapy, a fast high-level web crawling & scraping framework for Python.
Decode Bash ANSI-C escapes in Request.from_curl() (#8433)
Hygor luz2aefa609f1a9
diff --git a/scrapy/utils/curl.py b/scrapy/utils/curl.pyindex bc98fbec3..f040134d3 100644--- a/scrapy/utils/curl.py+++ b/scrapy/utils/curl.py@@ -1,9 +1,10 @@from __future__ import annotationsimport argparse+import reimport warningsfrom http.cookies import SimpleCookie-from shlex import split+from shlex import quote, splitfrom typing import TYPE_CHECKING, Any, NoReturnfrom urllib.parse import urlparse@@ -94,8 +95,63 @@ def _parse_headers_and_cookies(return headers, cookies+_ANSI_C_QUOTED_RE = re.compile(r"\$'((?:[^'\\]|\\.)*)'", re.DOTALL)+# https://www.gnu.org/software/bash/manual/html_node/ANSI_002dC-Quoting.html+_ANSI_C_ESCAPE_RE = re.compile(+ r"\\(?:([abeEfnrtv\\'\"?])|u([0-9A-Fa-f]{1,4})|U([0-9A-Fa-f]{1,8})|c(\\\\?|.))"+ # A run of octal and hex escapes, each of them a byte.+ r"|((?:\\(?:[0-7]{1,3}|x[0-9A-Fa-f]{1,2}))+)",+ re.DOTALL,+)+_ANSI_C_SIMPLE_ESCAPES = {+ "a": "\a",+ "b": "\b",+ "e": "\x1b",+ "E": "\x1b",+ "f": "\f",+ "n": "\n",+ "r": "\r",+ "t": "\t",+ "v": "\v",+}+# Maps the surrogates that the surrogateescape error handler uses for+# undecodable bytes to the Latin-1 characters of those bytes.+_SURROGATE_ESCAPES_TO_LATIN1 = {0xDC00 + byte: byte for byte in range(0x80, 0x100)}+++def _decode_ansi_c_escape(match: re.Match[str]) -> str:+ simple, short_unicode, long_unicode, control, byte_escapes = match.groups()+ if simple is not None:+ return _ANSI_C_SIMPLE_ESCAPES.get(simple, simple)+ if control is not None:+ # \c\\ counts as \c followed by a single backslash.+ return "\x7f" if control == "?" else chr(ord(control[0]) & 0x1F)+ if byte_escapes is not None:+ data = bytes(+ int(escape[1:], 16) if escape[0] == "x" else int(escape, 8) & 0xFF+ for escape in byte_escapes.split("\\")[1:]+ )+ # Bash (printf %q) writes non-ASCII characters as their UTF-8 bytes,+ # while Firefox writes characters below U+0100 as \xHH, so bytes that+ # are not valid UTF-8 are read as Latin-1.+ return data.decode("utf-8", "surrogateescape").translate(+ _SURROGATE_ESCAPES_TO_LATIN1+ )+ return chr(int(short_unicode or long_unicode, 16))+++def _decode_ansi_c_string(value: str) -> str:+ """Decode *value*, the contents of a Bash ``$'...'`` string."""+ decoded = _ANSI_C_ESCAPE_RE.sub(_decode_ansi_c_escape, value)+ # Bash cuts the string at the first NUL byte.+ decoded = decoded.partition("\0")[0]+ # Firefox writes characters above U+FFFF as a UTF-16 surrogate pair of+ # \u escapes; join them, and fail on unpaired surrogates.+ return decoded.encode("utf-16-le", "surrogatepass").decode("utf-16-le")++def _normalize_curl_command(curl_command: str) -> str:- """Normalize Bash ANSI-C quote prefixes for shlex."""+ """Replace Bash ANSI-C quoted strings with their value quoted for shlex."""result: list[str] = []in_single_quote = Falsein_double_quote = False@@ -119,14 +175,13 @@ def _normalize_curl_command(curl_command: str) -> str:result.append(char)elif (char == "$"- and i + 1 < len(curl_command)- and curl_command[i + 1] == "'"and not in_single_quoteand not in_double_quote+ and (match := _ANSI_C_QUOTED_RE.match(curl_command, i))):- result.append("'")- in_single_quote = True- i += 1+ result.append(quote(_decode_ansi_c_string(match[1])))+ i = match.end()+ continueelse:result.append(char)diff --git a/tests/test_utils_curl.py b/tests/test_utils_curl.pyindex 38c1c3f5a..17809adb8 100644--- a/tests/test_utils_curl.py+++ b/tests/test_utils_curl.py@@ -296,6 +296,59 @@ class TestCurlToRequestKwargs:}self._test_command(curl_command, expected_result)+ @pytest.mark.parametrize(+ ("data", "body"),+ [+ (r"$'line1\nline2'", "line1\nline2"),+ (r"""$'{"name":"O\'Brien"}'""", """{"name":"O'Brien"}"""),+ (r"""$'\a\b\e\E\f\n\r\t\v\\\'\"\?'""", "\a\b\x1b\x1b\f\n\r\t\v\\'\"?"),+ (r"$'\101\0101\1012'", "A\x081A2"),+ (r"$'\303\251'", "é"),+ (r"$'\x41\x4G\x414'", "A\x04GA4"),+ (r"$'\u00e9\u20AC\u41G'", "é€AG"),+ (r"$'\U0001F600\U41'", "\U0001f600A"),+ (+ r"""$'{"name":"O\'Brien","city":"Z\xfcrich \ud83d\ude00"}'""",+ '{"name":"O\'Brien","city":"Zürich \U0001f600"}',+ ),+ (r"$'caf\xe9'", "café"),+ (r"$'\303\251\xe9'", "éé"),+ (r"$'\xff'", "ÿ"),+ (r"$'\ca\cZ\c[\c?\c\\x'", "\x01\x1a\x1b\x7f\x1cx"),+ (r"$'\z\q\8'", r"\z\q\8"),+ (r"$'\x'", r"\x"),+ (r"$'\u'", r"\u"),+ (r"$'\c'", r"\c"),+ (r"x$'a\0b'y", "xay"),+ (r"$'A\400B'", "A"),+ ("a$'b c'd", "ab cd"),+ ("a$''b", "ab"),+ (r"\$'a\nb'", r"$a\nb"),+ (r'''"$'a\nb'"''', r"$'a\nb'"),+ ],+ )+ def test_post_data_raw_ansi_c_quoting(self, data, body):+ curl_command = f"curl 'https://www.example.org/' --data-raw {data}"+ expected_result = {+ "method": "POST",+ "url": "https://www.example.org/",+ "body": body,+ }+ self._test_command(curl_command, expected_result)++ @pytest.mark.parametrize(+ ("data", "match"),+ [+ ("$'abc", "No closing quotation"),+ (r"$'\ud800'", "can't decode"),+ (r"$'\ude00\ud83d'", "can't decode"),+ (r"$'\U00110000'", r"chr\(\) arg not in range"),+ ],+ )+ def test_post_data_raw_ansi_c_quoting_error(self, data, match):+ with pytest.raises(ValueError, match=match):+ curl_to_request_kwargs(f"curl 'https://www.example.org/' --data-raw {data}")+def test_explicit_get_with_data(self):curl_command = "curl httpbin.org/anything -X GET --data asdf"expected_result = {