Scrapy, a fast high-level web crawling & scraping framework for Python.
from __future__ import annotations
from typing import Any
from urllib.parse import urlparse
import pytest
from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.http import Request, Response
from scrapy.linkextractors import IGNORED_EXTENSIONS
from scrapy.spiders import Spider
from scrapy.utils.url import (
_is_filesystem_path,
add_http_if_no_scheme,
guess_scheme,
strip_url,
url_has_any_extension,
url_is_from_any_domain,
url_is_from_spider,
)
def test_url_is_from_any_domain():
url = "http://www.wheele-bin-art.co.uk/get/product/123"
assert url_is_from_any_domain(url, ["wheele-bin-art.co.uk"])
assert not url_is_from_any_domain(url, ["art.co.uk"])
url = "http://wheele-bin-art.co.uk/get/product/123"
assert url_is_from_any_domain(url, ["wheele-bin-art.co.uk"])
assert not url_is_from_any_domain(url, ["art.co.uk"])
url = "http://www.Wheele-Bin-Art.co.uk/get/product/123"
assert url_is_from_any_domain(url, ["wheele-bin-art.CO.UK"])
assert url_is_from_any_domain(url, ["WHEELE-BIN-ART.CO.UK"])
url = "http://192.169.0.15:8080/mypage.html"
assert url_is_from_any_domain(url, ["192.169.0.15:8080"])
assert not url_is_from_any_domain(url, ["192.169.0.15"])
url = (
"javascript:%20document.orderform_2581_1190810811.mode.value=%27add%27;%20"
"javascript:%20document.orderform_2581_1190810811.submit%28%29"
)
assert not url_is_from_any_domain(url, ["testdomain.com"])
assert not url_is_from_any_domain(url + ".testdomain.com", ["testdomain.com"])
@pytest.mark.parametrize(
"url",
[
"https://www.example.com/page",
b"https://www.example.com/page",
urlparse("https://www.example.com/page"),
Request("https://www.example.com/page"),
Response("https://www.example.com/page"),
],
ids=["str", "bytes", "ParseResult", "Request", "Response"],
)
def test_url_is_from_any_domain_input_types(url: Any) -> None:
assert url_is_from_any_domain(url, ["example.com"])
assert not url_is_from_any_domain(url, ["example.org"])
def test_url_is_from_spider():
class MySpider(Spider):
name = "example.com"
assert url_is_from_spider("http://www.example.com/some/page.html", MySpider)
assert url_is_from_spider("http://sub.example.com/some/page.html", MySpider)
assert not url_is_from_spider("http://www.example.org/some/page.html", MySpider)
assert not url_is_from_spider("http://www.example.net/some/page.html", MySpider)
def test_url_is_from_spider_with_allowed_domains():
class MySpider(Spider):
name = "example.com"
allowed_domains = ["example.org", "example.net"]
assert url_is_from_spider("http://www.example.com/some/page.html", MySpider)
assert url_is_from_spider("http://sub.example.com/some/page.html", MySpider)
assert url_is_from_spider("http://example.com/some/page.html", MySpider)
assert url_is_from_spider("http://www.example.org/some/page.html", MySpider)
assert url_is_from_spider("http://www.example.net/some/page.html", MySpider)
assert not url_is_from_spider("http://www.example.us/some/page.html", MySpider)
class MySpider2(Spider):
name = "example.com"
allowed_domains = {"example.com", "example.net"}
assert url_is_from_spider("http://www.example.com/some/page.html", MySpider2)
class MySpider3(Spider):
name = "example.com"
allowed_domains = ("example.com", "example.net")
assert url_is_from_spider("http://www.example.com/some/page.html", MySpider3)
def test_url_is_from_spider_with_allowed_domains_property():
class MySpider(Spider):
name = "myspider"
@property
def allowed_domains(self): # type: ignore[override]
return ["example.org"]
with pytest.warns(UserWarning, match="allowed_domains is a property"):
assert not url_is_from_spider("http://www.example.org/some/page.html", MySpider)
@pytest.mark.parametrize(
("url", "expected"),
[
("http://www.example.com/archive.tar.gz", True),
("http://www.example.com/page.doc", True),
("http://www.example.com/page.pdf", True),
("http://www.example.com/page.htm", False),
("http://www.example.com/", False),
("http://www.example.com/page.doc.html", False),
],
)
def test_url_has_any_extension(url: str, expected: bool) -> None:
deny_extensions = {"." + e for e in IGNORED_EXTENSIONS}
assert url_has_any_extension(url, deny_extensions) is expected
def test_add_http_if_no_scheme_deprecated() -> None:
with pytest.warns(
ScrapyDeprecationWarning, match="use w3lib.url.add_http_if_no_scheme"
):
assert add_http_if_no_scheme("www.example.com") == "http://www.example.com"
@pytest.mark.parametrize(
("url", "expected"),
[
("/index", "file://"),
("/index.html", "file://"),
("./index.html", "file://"),
("../index.html", "file://"),
("../../index.html", "file://"),
("./data/index.html", "file://"),
(".hidden/data/index.html", "file://"),
("/home/user/www/index.html", "file://"),
("//home/user/www/index.html", "file://"),
("file:///home/user/www/index.html", "file://"),
("index.html", "http://"),
("example.com", "http://"),
("www.example.com", "http://"),
("www.example.com/index.html", "http://"),
("http://example.com", "http://"),
("http://example.com/index.html", "http://"),
("localhost", "http://"),
("localhost/index.html", "http://"),
# some corner cases (default to http://)
("/", "http://"),
(".../test", "http://"),
],
)
def test_guess_scheme(url: str, expected: str):
assert guess_scheme(url).startswith(expected)
@pytest.mark.parametrize(
("url", "expected", "reason"),
[
(
r"C:\absolute\path\to\a\file.html",
"file://",
"Windows filepath are not supported for scrapy shell",
),
],
)
def test_guess_scheme_skipped(url: str, expected: str, reason: str) -> None:
pytest.skip(reason)
class TestStripUrl:
@pytest.mark.parametrize(
"url",
[
"http://www.example.com/index.html",
"http://www.example.com/index.html?somekey=somevalue",
],
)
def test_noop(self, url: str) -> None:
assert strip_url(url) == url
def test_fragments(self):
assert (
strip_url(
"http://www.example.com/index.html?somekey=somevalue#section",
strip_fragment=False,
)
== "http://www.example.com/index.html?somekey=somevalue#section"
)
@pytest.mark.parametrize(
("url", "origin", "expected"),
[
("http://www.example.com/", False, "http://www.example.com/"),
("http://www.example.com", False, "http://www.example.com"),
("http://www.example.com", True, "http://www.example.com/"),
],
)
def test_path(self, url: str, origin: bool, expected: str) -> None:
assert strip_url(url, origin_only=origin) == expected
@pytest.mark.parametrize(
("url", "expected"),
[
(
"http://[email protected]/index.html?somekey=somevalue#section",
"http://www.example.com/index.html?somekey=somevalue",
),
(
"https://username:@www.example.com/index.html?somekey=somevalue#section",
"https://www.example.com/index.html?somekey=somevalue",
),
(
"ftp://username:[email protected]/index.html?somekey=somevalue#section",
"ftp://www.example.com/index.html?somekey=somevalue",
),
# user: "username@", password: none
(
"http://username%[email protected]/index.html?somekey=somevalue#section",
"http://www.example.com/index.html?somekey=somevalue",
),
# user: "username:pass", password: ""
(
"https://username%3Apass:@www.example.com/index.html?somekey=somevalue#section",
"https://www.example.com/index.html?somekey=somevalue",
),
# user: "me", password: "[email protected]"
(
"ftp://me:user%40d[email protected]/index.html?somekey=somevalue#section",
"ftp://www.example.com/index.html?somekey=somevalue",
),
],
)
def test_credentials(self, url: str, expected: str) -> None:
assert strip_url(url, strip_credentials=True) == expected
@pytest.mark.parametrize(
("url", "expected"),
[
(
"http://username:[email protected]:80/index.html?somekey=somevalue#section",
"http://www.example.com/index.html?somekey=somevalue",
),
(
"http://username:[email protected]:8080/index.html#section",
"http://www.example.com:8080/index.html",
),
(
"http://username:[email protected]:443/index.html?somekey=somevalue&someotherkey=sov#section",
"http://www.example.com:443/index.html?somekey=somevalue&someotherkey=sov",
),
(
"https://username:[email protected]:443/index.html",
"https://www.example.com/index.html",
),
(
"https://username:[email protected]:442/index.html",
"https://www.example.com:442/index.html",
),
(
"https://username:[email protected]:80/index.html",
"https://www.example.com:80/index.html",
),
(
"ftp://username:[email protected]:21/file.txt",
"ftp://www.example.com/file.txt",
),
(
"ftp://username:[email protected]:221/file.txt",
"ftp://www.example.com:221/file.txt",
),
],
)
def test_default_ports_creds_off(self, url: str, expected: str) -> None:
assert strip_url(url) == expected
@pytest.mark.parametrize(
("url", "expected"),
[
(
"http://username:[email protected]:80/index.html",
"http://username:[email protected]/index.html",
),
(
"http://username:[email protected]:8080/index.html",
"http://username:[email protected]:8080/index.html",
),
(
"http://username:[email protected]:443/index.html",
"http://username:[email protected]:443/index.html",
),
(
"https://username:[email protected]:443/index.html",
"https://username:[email protected]/index.html",
),
(
"https://username:[email protected]:442/index.html",
"https://username:[email protected]:442/index.html",
),
(
"https://username:[email protected]:80/index.html",
"https://username:[email protected]:80/index.html",
),
(
"ftp://username:[email protected]:21/file.txt",
"ftp://username:[email protected]/file.txt",
),
(
"ftp://username:[email protected]:221/file.txt",
"ftp://username:[email protected]:221/file.txt",
),
(
"http://user:[email protected]:80/index.html",
"http://user:[email protected]/index.html",
),
(
"https://user:[email protected]:443/index.html",
"https://user:[email protected]/index.html",
),
(
"ftp://user:[email protected]:21/file.txt",
"ftp://user:[email protected]/file.txt",
),
],
)
def test_default_ports(self, url: str, expected: str) -> None:
assert (
strip_url(url, strip_default_port=True, strip_credentials=False) == expected
)
@pytest.mark.parametrize(
("url", "expected"),
[
(
"http://username:[email protected]:80/index.html?somekey=somevalue&someotherkey=sov#section",
"http://username:[email protected]:80/index.html?somekey=somevalue&someotherkey=sov",
),
(
"http://username:[email protected]:8080/index.html?somekey=somevalue&someotherkey=sov#section",
"http://username:[email protected]:8080/index.html?somekey=somevalue&someotherkey=sov",
),
(
"http://username:[email protected]:443/index.html",
"http://username:[email protected]:443/index.html",
),
(
"https://username:[email protected]:443/index.html",
"https://username:[email protected]:443/index.html",
),
(
"https://username:[email protected]:442/index.html",
"https://username:[email protected]:442/index.html",
),
(
"https://username:[email protected]:80/index.html",
"https://username:[email protected]:80/index.html",
),
(
"ftp://username:[email protected]:21/file.txt",
"ftp://username:[email protected]:21/file.txt",
),
(
"ftp://username:[email protected]:221/file.txt",
"ftp://username:[email protected]:221/file.txt",
),
],
)
def test_default_ports_keep(self, url: str, expected: str) -> None:
assert (
strip_url(url, strip_default_port=False, strip_credentials=False)
== expected
)
@pytest.mark.parametrize(
("url", "expected"),
[
(
"http://username:[email protected]/index.html",
"http://www.example.com/",
),
(
"http://username:[email protected]:80/foo/bar?query=value#somefrag",
"http://www.example.com/",
),
(
"http://username:[email protected]:8008/foo/bar?query=value#somefrag",
"http://www.example.com:8008/",
),
(
"https://username:[email protected]:443/index.html",
"https://www.example.com/",
),
],
)
def test_origin_only(self, url: str, expected: str) -> None:
assert strip_url(url, origin_only=True) == expected
@pytest.mark.parametrize(
("path", "expected"),
[
# https://en.wikipedia.org/wiki/Path_(computing)#Representations_of_paths_by_operating_system_and_shell
# Unix-like OS, Microsoft Windows / cmd.exe
("/home/user/docs/Letter.txt", True),
("./inthisdir", True),
("../../greatgrandparent", True),
("~/.rcinfo", True),
(r"C:\user\docs\Letter.txt", True),
("/user/docs/Letter.txt", True),
(r"C:\Letter.txt", True),
(r"\\Server01\user\docs\Letter.txt", True),
(r"\\?\UNC\Server01\user\docs\Letter.txt", True),
(r"\\?\C:\user\docs\Letter.txt", True),
(r"C:\user\docs\somefile.ext:alternate_stream_name", True),
(r"https://example.com", False),
],
)
def test__is_filesystem_path(path: str, expected: bool) -> None:
assert _is_filesystem_path(path) == expected