Scrapy, a fast high-level web crawling & scraping framework for Python.
from __future__ import annotations
import warnings
from typing import Any
import pytest
from w3lib.http import basic_auth_header
from scrapy import Request
from scrapy.utils.curl import curl_to_request_kwargs
class TestCurlToRequestKwargs:
@staticmethod
def _test_command(curl_command: str, expected_result: dict[str, Any]) -> None:
result = curl_to_request_kwargs(curl_command)
assert result == expected_result
try:
Request(**result)
except TypeError as e:
pytest.fail(f"Request kwargs are not correct {e}")
def test_get(self):
curl_command = "curl http://example.org/"
expected_result = {"method": "GET", "url": "http://example.org/"}
self._test_command(curl_command, expected_result)
def test_get_url_with_escaped_space(self):
curl_command = r"curl http://example.org/a\ b"
expected_result = {
"method": "GET",
"url": "http://example.org/a b",
}
self._test_command(curl_command, expected_result)
def test_get_without_scheme(self):
curl_command = "curl www.example.org"
expected_result = {"method": "GET", "url": "http://www.example.org"}
self._test_command(curl_command, expected_result)
def test_get_basic_auth(self):
curl_command = 'curl "https://api.test.com/" -u "some_username:some_password"'
expected_result = {
"method": "GET",
"url": "https://api.test.com/",
"headers": [
("Authorization", basic_auth_header("some_username", "some_password"))
],
}
self._test_command(curl_command, expected_result)
@pytest.mark.parametrize("auth", ["some_username", "some_username:"])
def test_get_basic_auth_without_password(self, auth):
curl_command = f'curl "https://api.test.com/" -u "{auth}"'
expected_result = {
"method": "GET",
"url": "https://api.test.com/",
"headers": [("Authorization", basic_auth_header("some_username", ""))],
}
self._test_command(curl_command, expected_result)
def test_get_header_with_empty_value(self):
curl_command = 'curl "http://example.org/" -H "X-Flag;"'
expected_result = {
"method": "GET",
"url": "http://example.org/",
"headers": [("X-Flag", "")],
}
self._test_command(curl_command, expected_result)
def test_get_header_with_string_prefix(self):
curl_command = "curl 'http://example.org/' -H $'X-A: b'"
expected_result = {
"method": "GET",
"url": "http://example.org/",
"headers": [("X-A", "b")],
}
self._test_command(curl_command, expected_result)
@pytest.mark.parametrize("header", ["X-Flag:", "X-Flag: "])
def test_get_header_without_value(self, header):
curl_command = f'curl "http://example.org/" -H "{header}"'
expected_result = {
"method": "GET",
"url": "http://example.org/",
"headers": [("X-Flag", None)],
}
self._test_command(curl_command, expected_result)
@pytest.mark.parametrize("header", ["X-Flag", "X-Flag;extra", "X-Flag ; ", ";"])
def test_get_header_curl_would_not_send(self, header):
curl_command = f'curl "http://example.org/" -H "{header}"'
expected_result = {"method": "GET", "url": "http://example.org/"}
self._test_command(curl_command, expected_result)
def test_get_ignored_header_does_not_hide_following_header(self):
curl_command = 'curl "http://example.org/" -H "X-Flag" -H "X-Other: value"'
expected_result = {
"method": "GET",
"url": "http://example.org/",
"headers": [("X-Other", "value")],
}
self._test_command(curl_command, expected_result)
def test_get_cookie_option(self):
curl_command = 'curl "http://example.org/" -b "a=1; b=2"'
expected_result = {
"method": "GET",
"url": "http://example.org/",
"cookies": {"a": "1", "b": "2"},
}
self._test_command(curl_command, expected_result)
def test_get_cookie_file_option(self):
# curl reads cookies from a file when the value is not a key-value
# pair; Scrapy ignores it.
curl_command = 'curl "http://example.org/" -b cookies.txt -b "a=1"'
expected_result = {
"method": "GET",
"url": "http://example.org/",
"cookies": {"a": "1"},
}
self._test_command(curl_command, expected_result)
def test_get_complex(self):
curl_command = (
"curl 'http://httpbin.org/get' -H 'Accept-Encoding: gzip, deflate'"
" -H 'Accept-Language: en-US,en;q=0.9,ru;q=0.8,es;q=0.7' -H 'Upgra"
"de-Insecure-Requests: 1' -H 'User-Agent: Mozilla/5.0 (X11; Linux "
"x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Ubuntu Chromium/62"
".0.3202.75 Chrome/62.0.3202.75 Safari/537.36' -H 'Accept: text/ht"
"ml,application/xhtml+xml,application/xml;q=0.9,image/webp,image/a"
"png,*/*;q=0.8' -H 'Referer: http://httpbin.org/' -H 'Cookie: _gau"
"ges_unique_year=1; _gauges_unique=1; _gauges_unique_month=1; _gau"
"ges_unique_hour=1' -H 'Connection: keep-alive' --compressed -b '_"
"gauges_unique_day=1'"
)
expected_result = {
"method": "GET",
"url": "http://httpbin.org/get",
"headers": [
("Accept-Encoding", "gzip, deflate"),
("Accept-Language", "en-US,en;q=0.9,ru;q=0.8,es;q=0.7"),
("Upgrade-Insecure-Requests", "1"),
(
"User-Agent",
(
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML"
", like Gecko) Ubuntu Chromium/62.0.3202.75 Chrome/62.0.32"
"02.75 Safari/537.36"
),
),
(
"Accept",
(
"text/html,application/xhtml+xml,application/xml;q=0.9,ima"
"ge/webp,image/apng,*/*;q=0.8"
),
),
("Referer", "http://httpbin.org/"),
("Connection", "keep-alive"),
],
"cookies": {
"_gauges_unique_year": "1",
"_gauges_unique_hour": "1",
"_gauges_unique_day": "1",
"_gauges_unique": "1",
"_gauges_unique_month": "1",
},
}
self._test_command(curl_command, expected_result)
def test_post(self):
curl_command = (
"curl 'http://httpbin.org/post' -X POST -H 'Cookie: _gauges_unique"
"_year=1; _gauges_unique=1; _gauges_unique_month=1; _gauges_unique"
"_hour=1; _gauges_unique_day=1' -H 'Origin: http://httpbin.org' -H"
" 'Accept-Encoding: gzip, deflate' -H 'Accept-Language: en-US,en;q"
"=0.9,ru;q=0.8,es;q=0.7' -H 'Upgrade-Insecure-Requests: 1' -H 'Use"
"r-Agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTM"
"L, like Gecko) Ubuntu Chromium/62.0.3202.75 Chrome/62.0.3202.75 S"
"afari/537.36' -H 'Content-Type: application/x-www-form-urlencoded"
"' -H 'Accept: text/html,application/xhtml+xml,application/xml;q=0"
".9,image/webp,image/apng,*/*;q=0.8' -H 'Cache-Control: max-age=0'"
" -H 'Referer: http://httpbin.org/forms/post' -H 'Connection: keep"
"-alive' --data 'custname=John+Smith&custtel=500&custemail=jsmith%"
"40example.org&size=small&topping=cheese&topping=onion&delivery=12"
"%3A15&comments=' --compressed"
)
expected_result = {
"method": "POST",
"url": "http://httpbin.org/post",
"body": "custname=John+Smith&custtel=500&custemail=jsmith%40exampl"
"e.org&size=small&topping=cheese&topping=onion&delivery=12"
"%3A15&comments=",
"cookies": {
"_gauges_unique_year": "1",
"_gauges_unique_hour": "1",
"_gauges_unique_day": "1",
"_gauges_unique": "1",
"_gauges_unique_month": "1",
},
"headers": [
("Origin", "http://httpbin.org"),
("Accept-Encoding", "gzip, deflate"),
("Accept-Language", "en-US,en;q=0.9,ru;q=0.8,es;q=0.7"),
("Upgrade-Insecure-Requests", "1"),
(
"User-Agent",
(
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML"
", like Gecko) Ubuntu Chromium/62.0.3202.75 Chrome/62.0.32"
"02.75 Safari/537.36"
),
),
("Content-Type", "application/x-www-form-urlencoded"),
(
"Accept",
(
"text/html,application/xhtml+xml,application/xml;q=0.9,ima"
"ge/webp,image/apng,*/*;q=0.8"
),
),
("Cache-Control", "max-age=0"),
("Referer", "http://httpbin.org/forms/post"),
("Connection", "keep-alive"),
],
}
self._test_command(curl_command, expected_result)
def test_post_data_raw(self):
curl_command = (
"curl 'https://www.example.org/' --data-raw 'excerptLength=200&ena"
"bleDidYouMean=true&sortCriteria=ffirstz32xnamez32x201740686%20asc"
"ending&queryFunctions=%5B%5D&rankingFunctions=%5B%5D'"
)
expected_result = {
"method": "POST",
"url": "https://www.example.org/",
"body": (
"excerptLength=200&enableDidYouMean=true&sortCriteria=ffirstz3"
"2xnamez32x201740686%20ascending&queryFunctions=%5B%5D&ranking"
"Functions=%5B%5D"
),
}
self._test_command(curl_command, expected_result)
def test_post_data_raw_with_string_prefix(self):
curl_command = "curl 'https://www.example.org/' --data-raw $'{\"$filters\":\"Filter\u0021\"}'"
expected_result = {
"method": "POST",
"url": "https://www.example.org/",
"body": '{"$filters":"Filter!"}',
}
self._test_command(curl_command, expected_result)
def test_post_data_raw_preserves_leading_dollar(self):
curl_command = "curl 'https://www.example.org/' --data-raw '$top=10&$skip=5'"
expected_result = {
"method": "POST",
"url": "https://www.example.org/",
"body": "$top=10&$skip=5",
}
self._test_command(curl_command, expected_result)
def test_post_data_multiple(self):
# curl merges repeated -d/--data/--data-raw options into a single body
# joined with "&"; scrapy must do the same, not keep only the last one.
curl_command = "curl 'https://www.example.org/' -d 'a=1' -d 'b=2' -d 'c=3'"
expected_result = {
"method": "POST",
"url": "https://www.example.org/",
"body": "a=1&b=2&c=3",
}
self._test_command(curl_command, expected_result)
def test_post_data_multiple_mixed_flag_names(self):
curl_command = (
"curl 'https://www.example.org/' --data 'a=1' --data-raw 'b=2' -d 'c=3'"
)
expected_result = {
"method": "POST",
"url": "https://www.example.org/",
"body": "a=1&b=2&c=3",
}
self._test_command(curl_command, expected_result)
def test_post_data_multiple_with_string_prefix(self):
# Bash $'...' quoting is normalized before parsing,
# so the prefix is not included when the values are merged.
curl_command = "curl 'https://www.example.org/' -d $'a=1' -d $'b=2'"
expected_result = {
"method": "POST",
"url": "https://www.example.org/",
"body": "a=1&b=2",
}
self._test_command(curl_command, expected_result)
@pytest.mark.parametrize(
("data", "body"),
[
(r"$'line1\nline2'", "line1\nline2"),
(r"""$'{"name":"O\'Brien"}'""", """{"name":"O'Brien"}"""),
(r"""$'\a\b\e\E\f\n\r\t\v\\\'\"\?'""", "\a\b\x1b\x1b\f\n\r\t\v\\'\"?"),
(r"$'\101\0101\1012'", "A\x081A2"),
(r"$'\303\251'", "é"),
(r"$'\x41\x4G\x414'", "A\x04GA4"),
(r"$'\u00e9\u20AC\u41G'", "é€AG"),
(r"$'\U0001F600\U41'", "\U0001f600A"),
(
r"""$'{"name":"O\'Brien","city":"Z\xfcrich \ud83d\ude00"}'""",
'{"name":"O\'Brien","city":"Zürich \U0001f600"}',
),
(r"$'caf\xe9'", "café"),
(r"$'\303\251\xe9'", "éé"),
(r"$'\xff'", "ÿ"),
(r"$'\ca\cZ\c[\c?\c\\x'", "\x01\x1a\x1b\x7f\x1cx"),
(r"$'\z\q\8'", r"\z\q\8"),
(r"$'\x'", r"\x"),
(r"$'\u'", r"\u"),
(r"$'\c'", r"\c"),
(r"x$'a\0b'y", "xay"),
(r"$'A\400B'", "A"),
("a$'b c'd", "ab cd"),
("a$''b", "ab"),
(r"\$'a\nb'", r"$a\nb"),
(r'''"$'a\nb'"''', r"$'a\nb'"),
],
)
def test_post_data_raw_ansi_c_quoting(self, data, body):
curl_command = f"curl 'https://www.example.org/' --data-raw {data}"
expected_result = {
"method": "POST",
"url": "https://www.example.org/",
"body": body,
}
self._test_command(curl_command, expected_result)
@pytest.mark.parametrize(
("data", "match"),
[
("$'abc", "No closing quotation"),
(r"$'\ud800'", "can't decode"),
(r"$'\ude00\ud83d'", "can't decode"),
(r"$'\U00110000'", r"chr\(\) arg not in range"),
],
)
def test_post_data_raw_ansi_c_quoting_error(self, data, match):
with pytest.raises(ValueError, match=match):
curl_to_request_kwargs(f"curl 'https://www.example.org/' --data-raw {data}")
def test_explicit_get_with_data(self):
curl_command = "curl httpbin.org/anything -X GET --data asdf"
expected_result = {
"method": "GET",
"url": "http://httpbin.org/anything",
"body": "asdf",
}
self._test_command(curl_command, expected_result)
def test_patch(self):
curl_command = (
'curl "https://example.com/api/fake" -u "username:password" -H "Ac'
'cept: application/vnd.go.cd.v4+json" -H "Content-Type: applicatio'
'n/json" -X PATCH -d \'{"hostname": "agent02.example.com", "agent'
'_config_state": "Enabled", "resources": ["Java","Linux"], "enviro'
'nments": ["Dev"]}\''
)
expected_result = {
"method": "PATCH",
"url": "https://example.com/api/fake",
"headers": [
("Accept", "application/vnd.go.cd.v4+json"),
("Content-Type", "application/json"),
("Authorization", basic_auth_header("username", "password")),
],
"body": '{"hostname": "agent02.example.com", "agent_config_state"'
': "Enabled", "resources": ["Java","Linux"], "environments'
'": ["Dev"]}',
}
self._test_command(curl_command, expected_result)
def test_delete(self):
curl_command = 'curl -X "DELETE" https://www.url.com/page'
expected_result = {"method": "DELETE", "url": "https://www.url.com/page"}
self._test_command(curl_command, expected_result)
def test_get_silent(self):
curl_command = 'curl --silent "www.example.com"'
expected_result = {"method": "GET", "url": "http://www.example.com"}
assert curl_to_request_kwargs(curl_command) == expected_result
def test_too_few_arguments_error(self):
with pytest.raises(
ValueError,
match=r"too few arguments|the following arguments are required:\s*url",
):
curl_to_request_kwargs("curl")
def test_ignore_unknown_options(self):
# case 1: ignore_unknown_options=True:
curl_command = "curl --bar --baz http://www.example.com"
expected_result = {"method": "GET", "url": "http://www.example.com"}
with warnings.catch_warnings(): # avoid warning when executing tests
warnings.filterwarnings(
"ignore", category=UserWarning, message="Unrecognized options:"
)
assert curl_to_request_kwargs(curl_command) == expected_result
# case 2: ignore_unknown_options=False (raise exception):
with pytest.raises(ValueError, match=r"Unrecognized options:.*--bar.*--baz"):
curl_to_request_kwargs(
"curl --bar --baz http://www.example.com", ignore_unknown_options=False
)
def test_must_start_with_curl_error(self):
with pytest.raises(ValueError, match="A curl command must start"):
curl_to_request_kwargs("carl -X POST http://example.org")
def test_empty_command_error(self):
with pytest.raises(ValueError, match="A curl command must start"):
curl_to_request_kwargs("")