Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

15.7 KB
Raw Blame
from __future__ import annotations

from typing import Any
from urllib.parse import urlparse

import pytest

from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.http import Request, Response
from scrapy.linkextractors import IGNORED_EXTENSIONS
from scrapy.spiders import Spider
from scrapy.utils.url import (
    _is_filesystem_path,
    add_http_if_no_scheme,
    guess_scheme,
    strip_url,
    url_has_any_extension,
    url_is_from_any_domain,
    url_is_from_spider,
)


def test_url_is_from_any_domain():
    url = "http://www.wheele-bin-art.co.uk/get/product/123"
    assert url_is_from_any_domain(url, ["wheele-bin-art.co.uk"])
    assert not url_is_from_any_domain(url, ["art.co.uk"])

    url = "http://wheele-bin-art.co.uk/get/product/123"
    assert url_is_from_any_domain(url, ["wheele-bin-art.co.uk"])
    assert not url_is_from_any_domain(url, ["art.co.uk"])

    url = "http://www.Wheele-Bin-Art.co.uk/get/product/123"
    assert url_is_from_any_domain(url, ["wheele-bin-art.CO.UK"])
    assert url_is_from_any_domain(url, ["WHEELE-BIN-ART.CO.UK"])

    url = "http://192.169.0.15:8080/mypage.html"
    assert url_is_from_any_domain(url, ["192.169.0.15:8080"])
    assert not url_is_from_any_domain(url, ["192.169.0.15"])

    url = (
        "javascript:%20document.orderform_2581_1190810811.mode.value=%27add%27;%20"
        "javascript:%20document.orderform_2581_1190810811.submit%28%29"
    )
    assert not url_is_from_any_domain(url, ["testdomain.com"])
    assert not url_is_from_any_domain(url + ".testdomain.com", ["testdomain.com"])


@pytest.mark.parametrize(
    "url",
    [
        "https://www.example.com/page",
        b"https://www.example.com/page",
        urlparse("https://www.example.com/page"),
        Request("https://www.example.com/page"),
        Response("https://www.example.com/page"),
    ],
    ids=["str", "bytes", "ParseResult", "Request", "Response"],
)
def test_url_is_from_any_domain_input_types(url: Any) -> None:
    assert url_is_from_any_domain(url, ["example.com"])
    assert not url_is_from_any_domain(url, ["example.org"])


def test_url_is_from_spider():
    class MySpider(Spider):
        name = "example.com"

    assert url_is_from_spider("http://www.example.com/some/page.html", MySpider)
    assert url_is_from_spider("http://sub.example.com/some/page.html", MySpider)
    assert not url_is_from_spider("http://www.example.org/some/page.html", MySpider)
    assert not url_is_from_spider("http://www.example.net/some/page.html", MySpider)


def test_url_is_from_spider_with_allowed_domains():
    class MySpider(Spider):
        name = "example.com"
        allowed_domains = ["example.org", "example.net"]

    assert url_is_from_spider("http://www.example.com/some/page.html", MySpider)
    assert url_is_from_spider("http://sub.example.com/some/page.html", MySpider)
    assert url_is_from_spider("http://example.com/some/page.html", MySpider)
    assert url_is_from_spider("http://www.example.org/some/page.html", MySpider)
    assert url_is_from_spider("http://www.example.net/some/page.html", MySpider)
    assert not url_is_from_spider("http://www.example.us/some/page.html", MySpider)

    class MySpider2(Spider):
        name = "example.com"
        allowed_domains = {"example.com", "example.net"}

    assert url_is_from_spider("http://www.example.com/some/page.html", MySpider2)

    class MySpider3(Spider):
        name = "example.com"
        allowed_domains = ("example.com", "example.net")

    assert url_is_from_spider("http://www.example.com/some/page.html", MySpider3)


def test_url_is_from_spider_with_allowed_domains_property():
    class MySpider(Spider):
        name = "myspider"

        @property
        def allowed_domains(self):  # type: ignore[override]
            return ["example.org"]

    with pytest.warns(UserWarning, match="allowed_domains is a property"):
        assert not url_is_from_spider("http://www.example.org/some/page.html", MySpider)


@pytest.mark.parametrize(
    ("url", "expected"),
    [
        ("http://www.example.com/archive.tar.gz", True),
        ("http://www.example.com/page.doc", True),
        ("http://www.example.com/page.pdf", True),
        ("http://www.example.com/page.htm", False),
        ("http://www.example.com/", False),
        ("http://www.example.com/page.doc.html", False),
    ],
)
def test_url_has_any_extension(url: str, expected: bool) -> None:
    deny_extensions = {"." + e for e in IGNORED_EXTENSIONS}
    assert url_has_any_extension(url, deny_extensions) is expected


def test_add_http_if_no_scheme_deprecated() -> None:
    with pytest.warns(
        ScrapyDeprecationWarning, match="use w3lib.url.add_http_if_no_scheme"
    ):
        assert add_http_if_no_scheme("www.example.com") == "http://www.example.com"


@pytest.mark.parametrize(
    ("url", "expected"),
    [
        ("/index", "file://"),
        ("/index.html", "file://"),
        ("./index.html", "file://"),
        ("../index.html", "file://"),
        ("../../index.html", "file://"),
        ("./data/index.html", "file://"),
        (".hidden/data/index.html", "file://"),
        ("/home/user/www/index.html", "file://"),
        ("//home/user/www/index.html", "file://"),
        ("file:///home/user/www/index.html", "file://"),
        ("index.html", "http://"),
        ("example.com", "http://"),
        ("www.example.com", "http://"),
        ("www.example.com/index.html", "http://"),
        ("http://example.com", "http://"),
        ("http://example.com/index.html", "http://"),
        ("localhost", "http://"),
        ("localhost/index.html", "http://"),
        # some corner cases (default to http://)
        ("/", "http://"),
        (".../test", "http://"),
    ],
)
def test_guess_scheme(url: str, expected: str):
    assert guess_scheme(url).startswith(expected)


@pytest.mark.parametrize(
    ("url", "expected", "reason"),
    [
        (
            r"C:\absolute\path\to\a\file.html",
            "file://",
            "Windows filepath are not supported for scrapy shell",
        ),
    ],
)
def test_guess_scheme_skipped(url: str, expected: str, reason: str) -> None:
    pytest.skip(reason)


class TestStripUrl:
    @pytest.mark.parametrize(
        "url",
        [
            "http://www.example.com/index.html",
            "http://www.example.com/index.html?somekey=somevalue",
        ],
    )
    def test_noop(self, url: str) -> None:
        assert strip_url(url) == url

    def test_fragments(self):
        assert (
            strip_url(
                "http://www.example.com/index.html?somekey=somevalue#section",
                strip_fragment=False,
            )
            == "http://www.example.com/index.html?somekey=somevalue#section"
        )

    @pytest.mark.parametrize(
        ("url", "origin", "expected"),
        [
            ("http://www.example.com/", False, "http://www.example.com/"),
            ("http://www.example.com", False, "http://www.example.com"),
            ("http://www.example.com", True, "http://www.example.com/"),
        ],
    )
    def test_path(self, url: str, origin: bool, expected: str) -> None:
        assert strip_url(url, origin_only=origin) == expected

    @pytest.mark.parametrize(
        ("url", "expected"),
        [
            (
                "http://[email protected]/index.html?somekey=somevalue#section",
                "http://www.example.com/index.html?somekey=somevalue",
            ),
            (
                "https://username:@www.example.com/index.html?somekey=somevalue#section",
                "https://www.example.com/index.html?somekey=somevalue",
            ),
            (
                "ftp://username:[email protected]/index.html?somekey=somevalue#section",
                "ftp://www.example.com/index.html?somekey=somevalue",
            ),
            # user: "username@", password: none
            (
                "http://username%[email protected]/index.html?somekey=somevalue#section",
                "http://www.example.com/index.html?somekey=somevalue",
            ),
            # user: "username:pass", password: ""
            (
                "https://username%3Apass:@www.example.com/index.html?somekey=somevalue#section",
                "https://www.example.com/index.html?somekey=somevalue",
            ),
            # user: "me", password: "[email protected]"
            (
                "ftp://me:user%40d[email protected]/index.html?somekey=somevalue#section",
                "ftp://www.example.com/index.html?somekey=somevalue",
            ),
        ],
    )
    def test_credentials(self, url: str, expected: str) -> None:
        assert strip_url(url, strip_credentials=True) == expected

    @pytest.mark.parametrize(
        ("url", "expected"),
        [
            (
                "http://username:[email protected]:80/index.html?somekey=somevalue#section",
                "http://www.example.com/index.html?somekey=somevalue",
            ),
            (
                "http://username:[email protected]:8080/index.html#section",
                "http://www.example.com:8080/index.html",
            ),
            (
                "http://username:[email protected]:443/index.html?somekey=somevalue&someotherkey=sov#section",
                "http://www.example.com:443/index.html?somekey=somevalue&someotherkey=sov",
            ),
            (
                "https://username:[email protected]:443/index.html",
                "https://www.example.com/index.html",
            ),
            (
                "https://username:[email protected]:442/index.html",
                "https://www.example.com:442/index.html",
            ),
            (
                "https://username:[email protected]:80/index.html",
                "https://www.example.com:80/index.html",
            ),
            (
                "ftp://username:[email protected]:21/file.txt",
                "ftp://www.example.com/file.txt",
            ),
            (
                "ftp://username:[email protected]:221/file.txt",
                "ftp://www.example.com:221/file.txt",
            ),
        ],
    )
    def test_default_ports_creds_off(self, url: str, expected: str) -> None:
        assert strip_url(url) == expected

    @pytest.mark.parametrize(
        ("url", "expected"),
        [
            (
                "http://username:[email protected]:80/index.html",
                "http://username:[email protected]/index.html",
            ),
            (
                "http://username:[email protected]:8080/index.html",
                "http://username:[email protected]:8080/index.html",
            ),
            (
                "http://username:[email protected]:443/index.html",
                "http://username:[email protected]:443/index.html",
            ),
            (
                "https://username:[email protected]:443/index.html",
                "https://username:[email protected]/index.html",
            ),
            (
                "https://username:[email protected]:442/index.html",
                "https://username:[email protected]:442/index.html",
            ),
            (
                "https://username:[email protected]:80/index.html",
                "https://username:[email protected]:80/index.html",
            ),
            (
                "ftp://username:[email protected]:21/file.txt",
                "ftp://username:[email protected]/file.txt",
            ),
            (
                "ftp://username:[email protected]:221/file.txt",
                "ftp://username:[email protected]:221/file.txt",
            ),
            (
                "http://user:[email protected]:80/index.html",
                "http://user:[email protected]/index.html",
            ),
            (
                "https://user:[email protected]:443/index.html",
                "https://user:[email protected]/index.html",
            ),
            (
                "ftp://user:[email protected]:21/file.txt",
                "ftp://user:[email protected]/file.txt",
            ),
        ],
    )
    def test_default_ports(self, url: str, expected: str) -> None:
        assert (
            strip_url(url, strip_default_port=True, strip_credentials=False) == expected
        )

    @pytest.mark.parametrize(
        ("url", "expected"),
        [
            (
                "http://username:[email protected]:80/index.html?somekey=somevalue&someotherkey=sov#section",
                "http://username:[email protected]:80/index.html?somekey=somevalue&someotherkey=sov",
            ),
            (
                "http://username:[email protected]:8080/index.html?somekey=somevalue&someotherkey=sov#section",
                "http://username:[email protected]:8080/index.html?somekey=somevalue&someotherkey=sov",
            ),
            (
                "http://username:[email protected]:443/index.html",
                "http://username:[email protected]:443/index.html",
            ),
            (
                "https://username:[email protected]:443/index.html",
                "https://username:[email protected]:443/index.html",
            ),
            (
                "https://username:[email protected]:442/index.html",
                "https://username:[email protected]:442/index.html",
            ),
            (
                "https://username:[email protected]:80/index.html",
                "https://username:[email protected]:80/index.html",
            ),
            (
                "ftp://username:[email protected]:21/file.txt",
                "ftp://username:[email protected]:21/file.txt",
            ),
            (
                "ftp://username:[email protected]:221/file.txt",
                "ftp://username:[email protected]:221/file.txt",
            ),
        ],
    )
    def test_default_ports_keep(self, url: str, expected: str) -> None:
        assert (
            strip_url(url, strip_default_port=False, strip_credentials=False)
            == expected
        )

    @pytest.mark.parametrize(
        ("url", "expected"),
        [
            (
                "http://username:[email protected]/index.html",
                "http://www.example.com/",
            ),
            (
                "http://username:[email protected]:80/foo/bar?query=value#somefrag",
                "http://www.example.com/",
            ),
            (
                "http://username:[email protected]:8008/foo/bar?query=value#somefrag",
                "http://www.example.com:8008/",
            ),
            (
                "https://username:[email protected]:443/index.html",
                "https://www.example.com/",
            ),
        ],
    )
    def test_origin_only(self, url: str, expected: str) -> None:
        assert strip_url(url, origin_only=True) == expected


@pytest.mark.parametrize(
    ("path", "expected"),
    [
        # https://en.wikipedia.org/wiki/Path_(computing)#Representations_of_paths_by_operating_system_and_shell
        # Unix-like OS, Microsoft Windows / cmd.exe
        ("/home/user/docs/Letter.txt", True),
        ("./inthisdir", True),
        ("../../greatgrandparent", True),
        ("~/.rcinfo", True),
        (r"C:\user\docs\Letter.txt", True),
        ("/user/docs/Letter.txt", True),
        (r"C:\Letter.txt", True),
        (r"\\Server01\user\docs\Letter.txt", True),
        (r"\\?\UNC\Server01\user\docs\Letter.txt", True),
        (r"\\?\C:\user\docs\Letter.txt", True),
        (r"C:\user\docs\somefile.ext:alternate_stream_name", True),
        (r"https://example.com", False),
    ],
)
def test__is_filesystem_path(path: str, expected: bool) -> None:
    assert _is_filesystem_path(path) == expected