Scrapy, a fast high-level web crawling & scraping framework for Python.
Add typing to scrapy/utils/url.py.
Andrey Rakhmatullinf64a7dedca6d
diff --git a/scrapy/utils/url.py b/scrapy/utils/url.pyindex 833aa3e20..22b4197f9 100644--- a/scrapy/utils/url.py+++ b/scrapy/utils/url.py@@ -6,6 +6,7 @@ Some of the functions that used to be imported from this module have been movedto the w3lib.url module. Always import those from there instead."""import re+from typing import TYPE_CHECKING, Iterable, Optional, Type, Union, castfrom urllib.parse import ParseResult, urldefrag, urlparse, urlunparse# scrapy.utils.url was moved to w3lib.url and import * ensures this@@ -15,8 +16,14 @@ from w3lib.url import _safe_chars, _unquotepath # noqa: F401from scrapy.utils.python import to_unicode+if TYPE_CHECKING:+ from scrapy import Spider-def url_is_from_any_domain(url, domains):++UrlT = Union[str, bytes, ParseResult]+++def url_is_from_any_domain(url: UrlT, domains: Iterable[str]) -> bool:"""Return True if the url belongs to any of the given domains"""host = parse_url(url).netloc.lower()if not host:@@ -25,29 +32,29 @@ def url_is_from_any_domain(url, domains):return any((host == d) or (host.endswith(f".{d}")) for d in domains)-def url_is_from_spider(url, spider):+def url_is_from_spider(url: UrlT, spider: Type["Spider"]) -> bool:"""Return True if the url belongs to the given spider"""return url_is_from_any_domain(url, [spider.name] + list(getattr(spider, "allowed_domains", [])))-def url_has_any_extension(url, extensions):+def url_has_any_extension(url: UrlT, extensions: Iterable[str]) -> bool:"""Return True if the url ends with one of the extensions provided"""lowercase_path = parse_url(url).path.lower()return any(lowercase_path.endswith(ext) for ext in extensions)-def parse_url(url, encoding=None):+def parse_url(url: UrlT, encoding: Optional[str] = None) -> ParseResult:"""Return urlparsed url from the given argument (which could be an alreadyparsed url)"""if isinstance(url, ParseResult):return url- return urlparse(to_unicode(url, encoding))+ return cast(ParseResult, urlparse(to_unicode(url, encoding)))-def escape_ajax(url):+def escape_ajax(url: str) -> str:"""Return the crawlable url according to:https://developers.google.com/webmasters/ajax-crawling/docs/getting-started@@ -76,7 +83,7 @@ def escape_ajax(url):return add_or_replace_parameter(defrag, "_escaped_fragment_", frag[1:])-def add_http_if_no_scheme(url):+def add_http_if_no_scheme(url: str) -> str:"""Add http as the default scheme if it is missing from the url."""match = re.match(r"^\w+://", url, flags=re.I)if not match:@@ -87,7 +94,7 @@ def add_http_if_no_scheme(url):return url-def _is_posix_path(string):+def _is_posix_path(string: str) -> bool:return bool(re.match(r"""@@ -109,7 +116,7 @@ def _is_posix_path(string):)-def _is_windows_path(string):+def _is_windows_path(string: str) -> bool:return bool(re.match(r"""@@ -125,11 +132,11 @@ def _is_windows_path(string):)-def _is_filesystem_path(string):+def _is_filesystem_path(string: str) -> bool:return _is_posix_path(string) or _is_windows_path(string)-def guess_scheme(url):+def guess_scheme(url: str) -> str:"""Add an URL scheme if missing: file:// for filepath-like input orhttp:// otherwise."""if _is_filesystem_path(url):@@ -138,12 +145,12 @@ def guess_scheme(url):def strip_url(- url,- strip_credentials=True,- strip_default_port=True,- origin_only=False,- strip_fragment=True,-):+ url: str,+ strip_credentials: bool = True,+ strip_default_port: bool = True,+ origin_only: bool = False,+ strip_fragment: bool = True,+) -> str:"""Strip URL string from some of its components:- ``strip_credentials`` removes "user:password@"