Scrapy, a fast high-level web crawling & scraping framework for Python.
Drop w3lib.url reexports.
Andrey Rakhmatullin5fc40f07f3c3
diff --git a/scrapy/utils/url.py b/scrapy/utils/url.pyindex 1c05ac20e..8f75e2618 100644--- a/scrapy/utils/url.py+++ b/scrapy/utils/url.py@@ -6,34 +6,10 @@ library.from __future__ import annotationsimport re-import warnings-from importlib import import_module-from typing import TYPE_CHECKING, Any, TypeAlias+from typing import TYPE_CHECKING, TypeAliasfrom urllib.parse import ParseResult, urlparse, urlunparse-from w3lib.url import __all__ as _public_w3lib_objects-from w3lib.url import any_to_uri as _any_to_uri-from w3lib.url import parse_url as _parse_url--from scrapy.exceptions import ScrapyDeprecationWarning--_DEPRECATED_NAMES: frozenset[str] = frozenset(- {"_unquotepath", "_safe_chars", "parse_url", *_public_w3lib_objects}-)---def __getattr__(name: str) -> Any:- if name in _DEPRECATED_NAMES:- obj_type = "attribute" if name == "_safe_chars" else "function"- warnings.warn(- f"The scrapy.utils.url.{name} {obj_type} is deprecated, use w3lib.url.{name} instead.",- ScrapyDeprecationWarning,- stacklevel=2,- )- return getattr(import_module("w3lib.url"), name)-- raise AttributeError-+from w3lib.url import any_to_uri, parse_urlif TYPE_CHECKING:from collections.abc import Iterable@@ -45,7 +21,7 @@ UrlT: TypeAlias = str | bytes | ParseResultdef url_is_from_any_domain(url: UrlT, domains: Iterable[str]) -> bool:"""Return True if the url belongs to any of the given domains"""- host = _parse_url(url).netloc.lower()+ host = parse_url(url).netloc.lower()if not host:return Falsereturn any((host == d) or (host.endswith(f".{d}")) for d in map(str.lower, domains))@@ -64,7 +40,7 @@ def url_is_from_spider(url: UrlT, spider: type[Spider]) -> bool:def url_has_any_extension(url: UrlT, extensions: Iterable[str]) -> bool:"""Return True if the url ends with one of the extensions provided"""- lowercase_path = _parse_url(url).path.lower()+ lowercase_path = parse_url(url).path.lower()return any(lowercase_path.endswith(ext) for ext in extensions)@@ -125,7 +101,7 @@ def guess_scheme(url: str) -> str:"""Add an URL scheme if missing: file:// for filepath-like input orhttp:// otherwise."""if _is_filesystem_path(url):- return _any_to_uri(url)+ return any_to_uri(url)return add_http_if_no_scheme(url)diff --git a/tests/test_utils_url.py b/tests/test_utils_url.pyindex bcbbe74a3..19a31c353 100644--- a/tests/test_utils_url.py+++ b/tests/test_utils_url.py@@ -1,13 +1,9 @@-import warnings-from importlib import import_module-import pytestfrom scrapy.linkextractors import IGNORED_EXTENSIONSfrom scrapy.spiders import Spider-from scrapy.utils.url import ( # type: ignore[attr-defined]+from scrapy.utils.url import (_is_filesystem_path,- _public_w3lib_objects,add_http_if_no_scheme,guess_scheme,strip_url,@@ -446,23 +442,3 @@ class TestStripUrl:)def test__is_filesystem_path(path: str, expected: bool) -> None:assert _is_filesystem_path(path) == expected--- "obj_name",- [- "_unquotepath",- "_safe_chars",- "parse_url",- *_public_w3lib_objects,- ],-)-def test_deprecated_imports_from_w3lib(obj_name: str) -> None:- with warnings.catch_warnings(record=True) as warns:- obj_type = "attribute" if obj_name == "_safe_chars" else "function"- message = f"The scrapy.utils.url.{obj_name} {obj_type} is deprecated, use w3lib.url.{obj_name} instead."-- getattr(import_module("scrapy.utils.url"), obj_name)-- assert isinstance(warns[0].message, Warning)- assert message in warns[0].message.args