Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

17.2 KB
Raw Blame
from __future__ import annotations

import warnings
from typing import Any

import pytest
from w3lib.http import basic_auth_header

from scrapy import Request
from scrapy.utils.curl import curl_to_request_kwargs


class TestCurlToRequestKwargs:
    @staticmethod
    def _test_command(curl_command: str, expected_result: dict[str, Any]) -> None:
        result = curl_to_request_kwargs(curl_command)
        assert result == expected_result
        try:
            Request(**result)
        except TypeError as e:
            pytest.fail(f"Request kwargs are not correct {e}")

    def test_get(self):
        curl_command = "curl http://example.org/"
        expected_result = {"method": "GET", "url": "http://example.org/"}
        self._test_command(curl_command, expected_result)

    def test_get_url_with_escaped_space(self):
        curl_command = r"curl http://example.org/a\ b"
        expected_result = {
            "method": "GET",
            "url": "http://example.org/a b",
        }
        self._test_command(curl_command, expected_result)

    def test_get_without_scheme(self):
        curl_command = "curl www.example.org"
        expected_result = {"method": "GET", "url": "http://www.example.org"}
        self._test_command(curl_command, expected_result)

    def test_get_basic_auth(self):
        curl_command = 'curl "https://api.test.com/" -u "some_username:some_password"'
        expected_result = {
            "method": "GET",
            "url": "https://api.test.com/",
            "headers": [
                ("Authorization", basic_auth_header("some_username", "some_password"))
            ],
        }
        self._test_command(curl_command, expected_result)

    @pytest.mark.parametrize("auth", ["some_username", "some_username:"])
    def test_get_basic_auth_without_password(self, auth):
        curl_command = f'curl "https://api.test.com/" -u "{auth}"'
        expected_result = {
            "method": "GET",
            "url": "https://api.test.com/",
            "headers": [("Authorization", basic_auth_header("some_username", ""))],
        }
        self._test_command(curl_command, expected_result)

    def test_get_header_with_empty_value(self):
        curl_command = 'curl "http://example.org/" -H "X-Flag;"'
        expected_result = {
            "method": "GET",
            "url": "http://example.org/",
            "headers": [("X-Flag", "")],
        }
        self._test_command(curl_command, expected_result)

    def test_get_header_with_string_prefix(self):
        curl_command = "curl 'http://example.org/' -H $'X-A: b'"
        expected_result = {
            "method": "GET",
            "url": "http://example.org/",
            "headers": [("X-A", "b")],
        }
        self._test_command(curl_command, expected_result)

    @pytest.mark.parametrize("header", ["X-Flag:", "X-Flag: "])
    def test_get_header_without_value(self, header):
        curl_command = f'curl "http://example.org/" -H "{header}"'
        expected_result = {
            "method": "GET",
            "url": "http://example.org/",
            "headers": [("X-Flag", None)],
        }
        self._test_command(curl_command, expected_result)

    @pytest.mark.parametrize("header", ["X-Flag", "X-Flag;extra", "X-Flag ; ", ";"])
    def test_get_header_curl_would_not_send(self, header):
        curl_command = f'curl "http://example.org/" -H "{header}"'
        expected_result = {"method": "GET", "url": "http://example.org/"}
        self._test_command(curl_command, expected_result)

    def test_get_ignored_header_does_not_hide_following_header(self):
        curl_command = 'curl "http://example.org/" -H "X-Flag" -H "X-Other: value"'
        expected_result = {
            "method": "GET",
            "url": "http://example.org/",
            "headers": [("X-Other", "value")],
        }
        self._test_command(curl_command, expected_result)

    def test_get_cookie_option(self):
        curl_command = 'curl "http://example.org/" -b "a=1; b=2"'
        expected_result = {
            "method": "GET",
            "url": "http://example.org/",
            "cookies": {"a": "1", "b": "2"},
        }
        self._test_command(curl_command, expected_result)

    def test_get_cookie_file_option(self):
        # curl reads cookies from a file when the value is not a key-value
        # pair; Scrapy ignores it.
        curl_command = 'curl "http://example.org/" -b cookies.txt -b "a=1"'
        expected_result = {
            "method": "GET",
            "url": "http://example.org/",
            "cookies": {"a": "1"},
        }
        self._test_command(curl_command, expected_result)

    def test_get_complex(self):
        curl_command = (
            "curl 'http://httpbin.org/get' -H 'Accept-Encoding: gzip, deflate'"
            " -H 'Accept-Language: en-US,en;q=0.9,ru;q=0.8,es;q=0.7' -H 'Upgra"
            "de-Insecure-Requests: 1' -H 'User-Agent: Mozilla/5.0 (X11; Linux "
            "x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Ubuntu Chromium/62"
            ".0.3202.75 Chrome/62.0.3202.75 Safari/537.36' -H 'Accept: text/ht"
            "ml,application/xhtml+xml,application/xml;q=0.9,image/webp,image/a"
            "png,*/*;q=0.8' -H 'Referer: http://httpbin.org/' -H 'Cookie: _gau"
            "ges_unique_year=1; _gauges_unique=1; _gauges_unique_month=1; _gau"
            "ges_unique_hour=1' -H 'Connection: keep-alive' --compressed -b '_"
            "gauges_unique_day=1'"
        )
        expected_result = {
            "method": "GET",
            "url": "http://httpbin.org/get",
            "headers": [
                ("Accept-Encoding", "gzip, deflate"),
                ("Accept-Language", "en-US,en;q=0.9,ru;q=0.8,es;q=0.7"),
                ("Upgrade-Insecure-Requests", "1"),
                (
                    "User-Agent",
                    (
                        "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML"
                        ", like Gecko) Ubuntu Chromium/62.0.3202.75 Chrome/62.0.32"
                        "02.75 Safari/537.36"
                    ),
                ),
                (
                    "Accept",
                    (
                        "text/html,application/xhtml+xml,application/xml;q=0.9,ima"
                        "ge/webp,image/apng,*/*;q=0.8"
                    ),
                ),
                ("Referer", "http://httpbin.org/"),
                ("Connection", "keep-alive"),
            ],
            "cookies": {
                "_gauges_unique_year": "1",
                "_gauges_unique_hour": "1",
                "_gauges_unique_day": "1",
                "_gauges_unique": "1",
                "_gauges_unique_month": "1",
            },
        }
        self._test_command(curl_command, expected_result)

    def test_post(self):
        curl_command = (
            "curl 'http://httpbin.org/post' -X POST -H 'Cookie: _gauges_unique"
            "_year=1; _gauges_unique=1; _gauges_unique_month=1; _gauges_unique"
            "_hour=1; _gauges_unique_day=1' -H 'Origin: http://httpbin.org' -H"
            " 'Accept-Encoding: gzip, deflate' -H 'Accept-Language: en-US,en;q"
            "=0.9,ru;q=0.8,es;q=0.7' -H 'Upgrade-Insecure-Requests: 1' -H 'Use"
            "r-Agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTM"
            "L, like Gecko) Ubuntu Chromium/62.0.3202.75 Chrome/62.0.3202.75 S"
            "afari/537.36' -H 'Content-Type: application/x-www-form-urlencoded"
            "' -H 'Accept: text/html,application/xhtml+xml,application/xml;q=0"
            ".9,image/webp,image/apng,*/*;q=0.8' -H 'Cache-Control: max-age=0'"
            " -H 'Referer: http://httpbin.org/forms/post' -H 'Connection: keep"
            "-alive' --data 'custname=John+Smith&custtel=500&custemail=jsmith%"
            "40example.org&size=small&topping=cheese&topping=onion&delivery=12"
            "%3A15&comments=' --compressed"
        )
        expected_result = {
            "method": "POST",
            "url": "http://httpbin.org/post",
            "body": "custname=John+Smith&custtel=500&custemail=jsmith%40exampl"
            "e.org&size=small&topping=cheese&topping=onion&delivery=12"
            "%3A15&comments=",
            "cookies": {
                "_gauges_unique_year": "1",
                "_gauges_unique_hour": "1",
                "_gauges_unique_day": "1",
                "_gauges_unique": "1",
                "_gauges_unique_month": "1",
            },
            "headers": [
                ("Origin", "http://httpbin.org"),
                ("Accept-Encoding", "gzip, deflate"),
                ("Accept-Language", "en-US,en;q=0.9,ru;q=0.8,es;q=0.7"),
                ("Upgrade-Insecure-Requests", "1"),
                (
                    "User-Agent",
                    (
                        "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML"
                        ", like Gecko) Ubuntu Chromium/62.0.3202.75 Chrome/62.0.32"
                        "02.75 Safari/537.36"
                    ),
                ),
                ("Content-Type", "application/x-www-form-urlencoded"),
                (
                    "Accept",
                    (
                        "text/html,application/xhtml+xml,application/xml;q=0.9,ima"
                        "ge/webp,image/apng,*/*;q=0.8"
                    ),
                ),
                ("Cache-Control", "max-age=0"),
                ("Referer", "http://httpbin.org/forms/post"),
                ("Connection", "keep-alive"),
            ],
        }
        self._test_command(curl_command, expected_result)

    def test_post_data_raw(self):
        curl_command = (
            "curl 'https://www.example.org/' --data-raw 'excerptLength=200&ena"
            "bleDidYouMean=true&sortCriteria=ffirstz32xnamez32x201740686%20asc"
            "ending&queryFunctions=%5B%5D&rankingFunctions=%5B%5D'"
        )
        expected_result = {
            "method": "POST",
            "url": "https://www.example.org/",
            "body": (
                "excerptLength=200&enableDidYouMean=true&sortCriteria=ffirstz3"
                "2xnamez32x201740686%20ascending&queryFunctions=%5B%5D&ranking"
                "Functions=%5B%5D"
            ),
        }
        self._test_command(curl_command, expected_result)

    def test_post_data_raw_with_string_prefix(self):
        curl_command = "curl 'https://www.example.org/' --data-raw $'{\"$filters\":\"Filter\u0021\"}'"
        expected_result = {
            "method": "POST",
            "url": "https://www.example.org/",
            "body": '{"$filters":"Filter!"}',
        }
        self._test_command(curl_command, expected_result)

    def test_post_data_raw_preserves_leading_dollar(self):
        curl_command = "curl 'https://www.example.org/' --data-raw '$top=10&$skip=5'"
        expected_result = {
            "method": "POST",
            "url": "https://www.example.org/",
            "body": "$top=10&$skip=5",
        }
        self._test_command(curl_command, expected_result)

    def test_post_data_multiple(self):
        # curl merges repeated -d/--data/--data-raw options into a single body
        # joined with "&"; scrapy must do the same, not keep only the last one.
        curl_command = "curl 'https://www.example.org/' -d 'a=1' -d 'b=2' -d 'c=3'"
        expected_result = {
            "method": "POST",
            "url": "https://www.example.org/",
            "body": "a=1&b=2&c=3",
        }
        self._test_command(curl_command, expected_result)

    def test_post_data_multiple_mixed_flag_names(self):
        curl_command = (
            "curl 'https://www.example.org/' --data 'a=1' --data-raw 'b=2' -d 'c=3'"
        )
        expected_result = {
            "method": "POST",
            "url": "https://www.example.org/",
            "body": "a=1&b=2&c=3",
        }
        self._test_command(curl_command, expected_result)

    def test_post_data_multiple_with_string_prefix(self):
        # Bash $'...' quoting is normalized before parsing,
        # so the prefix is not included when the values are merged.
        curl_command = "curl 'https://www.example.org/' -d $'a=1' -d $'b=2'"
        expected_result = {
            "method": "POST",
            "url": "https://www.example.org/",
            "body": "a=1&b=2",
        }
        self._test_command(curl_command, expected_result)

    @pytest.mark.parametrize(
        ("data", "body"),
        [
            (r"$'line1\nline2'", "line1\nline2"),
            (r"""$'{"name":"O\'Brien"}'""", """{"name":"O'Brien"}"""),
            (r"""$'\a\b\e\E\f\n\r\t\v\\\'\"\?'""", "\a\b\x1b\x1b\f\n\r\t\v\\'\"?"),
            (r"$'\101\0101\1012'", "A\x081A2"),
            (r"$'\303\251'", "é"),
            (r"$'\x41\x4G\x414'", "A\x04GA4"),
            (r"$'\u00e9\u20AC\u41G'", "é€AG"),
            (r"$'\U0001F600\U41'", "\U0001f600A"),
            (
                r"""$'{"name":"O\'Brien","city":"Z\xfcrich \ud83d\ude00"}'""",
                '{"name":"O\'Brien","city":"Zürich \U0001f600"}',
            ),
            (r"$'caf\xe9'", "café"),
            (r"$'\303\251\xe9'", "éé"),
            (r"$'\xff'", "ÿ"),
            (r"$'\ca\cZ\c[\c?\c\\x'", "\x01\x1a\x1b\x7f\x1cx"),
            (r"$'\z\q\8'", r"\z\q\8"),
            (r"$'\x'", r"\x"),
            (r"$'\u'", r"\u"),
            (r"$'\c'", r"\c"),
            (r"x$'a\0b'y", "xay"),
            (r"$'A\400B'", "A"),
            ("a$'b c'd", "ab cd"),
            ("a$''b", "ab"),
            (r"\$'a\nb'", r"$a\nb"),
            (r'''"$'a\nb'"''', r"$'a\nb'"),
        ],
    )
    def test_post_data_raw_ansi_c_quoting(self, data, body):
        curl_command = f"curl 'https://www.example.org/' --data-raw {data}"
        expected_result = {
            "method": "POST",
            "url": "https://www.example.org/",
            "body": body,
        }
        self._test_command(curl_command, expected_result)

    @pytest.mark.parametrize(
        ("data", "match"),
        [
            ("$'abc", "No closing quotation"),
            (r"$'\ud800'", "can't decode"),
            (r"$'\ude00\ud83d'", "can't decode"),
            (r"$'\U00110000'", r"chr\(\) arg not in range"),
        ],
    )
    def test_post_data_raw_ansi_c_quoting_error(self, data, match):
        with pytest.raises(ValueError, match=match):
            curl_to_request_kwargs(f"curl 'https://www.example.org/' --data-raw {data}")

    def test_explicit_get_with_data(self):
        curl_command = "curl httpbin.org/anything -X GET --data asdf"
        expected_result = {
            "method": "GET",
            "url": "http://httpbin.org/anything",
            "body": "asdf",
        }
        self._test_command(curl_command, expected_result)

    def test_patch(self):
        curl_command = (
            'curl "https://example.com/api/fake" -u "username:password" -H "Ac'
            'cept: application/vnd.go.cd.v4+json" -H "Content-Type: applicatio'
            'n/json" -X PATCH -d \'{"hostname": "agent02.example.com",  "agent'
            '_config_state": "Enabled", "resources": ["Java","Linux"], "enviro'
            'nments": ["Dev"]}\''
        )
        expected_result = {
            "method": "PATCH",
            "url": "https://example.com/api/fake",
            "headers": [
                ("Accept", "application/vnd.go.cd.v4+json"),
                ("Content-Type", "application/json"),
                ("Authorization", basic_auth_header("username", "password")),
            ],
            "body": '{"hostname": "agent02.example.com",  "agent_config_state"'
            ': "Enabled", "resources": ["Java","Linux"], "environments'
            '": ["Dev"]}',
        }
        self._test_command(curl_command, expected_result)

    def test_delete(self):
        curl_command = 'curl -X "DELETE" https://www.url.com/page'
        expected_result = {"method": "DELETE", "url": "https://www.url.com/page"}
        self._test_command(curl_command, expected_result)

    def test_get_silent(self):
        curl_command = 'curl --silent "www.example.com"'
        expected_result = {"method": "GET", "url": "http://www.example.com"}
        assert curl_to_request_kwargs(curl_command) == expected_result

    def test_too_few_arguments_error(self):
        with pytest.raises(
            ValueError,
            match=r"too few arguments|the following arguments are required:\s*url",
        ):
            curl_to_request_kwargs("curl")

    def test_ignore_unknown_options(self):
        # case 1: ignore_unknown_options=True:
        curl_command = "curl --bar --baz http://www.example.com"
        expected_result = {"method": "GET", "url": "http://www.example.com"}
        with warnings.catch_warnings():  # avoid warning when executing tests
            warnings.filterwarnings(
                "ignore", category=UserWarning, message="Unrecognized options:"
            )
            assert curl_to_request_kwargs(curl_command) == expected_result

        # case 2: ignore_unknown_options=False (raise exception):
        with pytest.raises(ValueError, match=r"Unrecognized options:.*--bar.*--baz"):
            curl_to_request_kwargs(
                "curl --bar --baz http://www.example.com", ignore_unknown_options=False
            )

    def test_must_start_with_curl_error(self):
        with pytest.raises(ValueError, match="A curl command must start"):
            curl_to_request_kwargs("carl -X POST http://example.org")

    def test_empty_command_error(self):
        with pytest.raises(ValueError, match="A curl command must start"):
            curl_to_request_kwargs("")