Scrapy, a fast high-level web crawling & scraping framework for Python.
Deprecate w3lib objects importable from scrapy.utils.url (#6586)
Laerte Pereiraa195af304d28
diff --git a/scrapy/utils/url.py b/scrapy/utils/url.pyindex 1cbfbfd99..3bf831c26 100644--- a/scrapy/utils/url.py+++ b/scrapy/utils/url.py@@ -1,36 +1,47 @@"""This module contains general purpose URL functions not found in the standardlibrary.--Some of the functions that used to be imported from this module have been moved-to the w3lib.url module. Always import those from there instead."""from __future__ import annotationsimport re-from typing import TYPE_CHECKING, Union, cast+import warnings+from importlib import import_module+from typing import TYPE_CHECKING, Unionfrom urllib.parse import ParseResult, urldefrag, urlparse, urlunparse-# scrapy.utils.url was moved to w3lib.url and import * ensures this-# move doesn't break old code-from w3lib.url import * # pylint: disable=unused-wildcard-import,wildcard-import-from w3lib.url import _safe_chars, _unquotepath # noqa: F401+from w3lib.url import __all__ as _public_w3lib_objects+from w3lib.url import add_or_replace_parameter as _add_or_replace_parameter+from w3lib.url import any_to_uri as _any_to_uri+from w3lib.url import parse_url as _parse_url++from scrapy.exceptions import ScrapyDeprecationWarning+++def __getattr__(name: str):+ if name in ("_unquotepath", "_safe_chars", "parse_url", *_public_w3lib_objects):+ obj_type = "attribute" if name == "_safe_chars" else "function"+ warnings.warn(+ f"The scrapy.utils.url.{name} {obj_type} is deprecated, use w3lib.url.{name} instead.",+ ScrapyDeprecationWarning,+ )+ return getattr(import_module("w3lib.url"), name)++ raise AttributeError-from scrapy.utils.python import to_unicodeif TYPE_CHECKING:from collections.abc import Iterablefrom scrapy import Spider-UrlT = Union[str, bytes, ParseResult]def url_is_from_any_domain(url: UrlT, domains: Iterable[str]) -> bool:"""Return True if the url belongs to any of the given domains"""- host = parse_url(url).netloc.lower()+ host = _parse_url(url).netloc.lower()if not host:return Falsedomains = [d.lower() for d in domains]@@ -46,21 +57,10 @@ def url_is_from_spider(url: UrlT, spider: type[Spider]) -> bool:def url_has_any_extension(url: UrlT, extensions: Iterable[str]) -> bool:"""Return True if the url ends with one of the extensions provided"""- lowercase_path = parse_url(url).path.lower()+ lowercase_path = _parse_url(url).path.lower()return any(lowercase_path.endswith(ext) for ext in extensions)-def parse_url( # pylint: disable=function-redefined- url: UrlT, encoding: str | None = None-) -> ParseResult:- """Return urlparsed url from the given argument (which could be an already- parsed url)- """- if isinstance(url, ParseResult):- return url- return cast(ParseResult, urlparse(to_unicode(url, encoding)))--def escape_ajax(url: str) -> str:"""Return the crawlable url@@ -86,7 +86,7 @@ def escape_ajax(url: str) -> str:defrag, frag = urldefrag(url)if not frag.startswith("!"):return url- return add_or_replace_parameter(defrag, "_escaped_fragment_", frag[1:])+ return _add_or_replace_parameter(defrag, "_escaped_fragment_", frag[1:])def add_http_if_no_scheme(url: str) -> str:@@ -146,7 +146,7 @@ def guess_scheme(url: str) -> str:"""Add an URL scheme if missing: file:// for filepath-like input orhttp:// otherwise."""if _is_filesystem_path(url):- return any_to_uri(url)+ return _any_to_uri(url)return add_http_if_no_scheme(url)diff --git a/tests/test_utils_url.py b/tests/test_utils_url.pyindex a15ad749d..62e2b5c1e 100644--- a/tests/test_utils_url.py+++ b/tests/test_utils_url.py@@ -1,10 +1,14 @@import unittest+import warnings++import pytestfrom scrapy.linkextractors import IGNORED_EXTENSIONSfrom scrapy.spiders import Spiderfrom scrapy.utils.misc import arg_to_iterfrom scrapy.utils.url import (_is_filesystem_path,+ _public_w3lib_objects,add_http_if_no_scheme,guess_scheme,strip_url,@@ -607,5 +611,26 @@ class IsPathTestCase(unittest.TestCase):)+ "obj_name",+ [+ "_unquotepath",+ "_safe_chars",+ "parse_url",+ *_public_w3lib_objects,+ ],+)+def test_deprecated_imports_from_w3lib(obj_name):+ with warnings.catch_warnings(record=True) as warns:+ obj_type = "attribute" if obj_name == "_safe_chars" else "function"+ message = f"The scrapy.utils.url.{obj_name} {obj_type} is deprecated, use w3lib.url.{obj_name} instead."++ from importlib import import_module++ getattr(import_module("scrapy.utils.url"), obj_name)++ assert message in warns[0].message.args++if __name__ == "__main__":unittest.main()