Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Deprecate w3lib objects importable from scrapy.utils.url (#6586)

Laerte Pereiraa195af304d28
diff --git a/scrapy/utils/url.py b/scrapy/utils/url.py
index 1cbfbfd99..3bf831c26 100644
--- a/scrapy/utils/url.py
+++ b/scrapy/utils/url.py
@@ -1,36 +1,47 @@
"""
This module contains general purpose URL functions not found in the standard
library.
-
-Some of the functions that used to be imported from this module have been moved
-to the w3lib.url module. Always import those from there instead.
"""
from __future__ import annotations
import re
-from typing import TYPE_CHECKING, Union, cast
+import warnings
+from importlib import import_module
+from typing import TYPE_CHECKING, Union
from urllib.parse import ParseResult, urldefrag, urlparse, urlunparse
-# scrapy.utils.url was moved to w3lib.url and import * ensures this
-# move doesn't break old code
-from w3lib.url import * # pylint: disable=unused-wildcard-import,wildcard-import
-from w3lib.url import _safe_chars, _unquotepath # noqa: F401
+from w3lib.url import __all__ as _public_w3lib_objects
+from w3lib.url import add_or_replace_parameter as _add_or_replace_parameter
+from w3lib.url import any_to_uri as _any_to_uri
+from w3lib.url import parse_url as _parse_url
+
+from scrapy.exceptions import ScrapyDeprecationWarning
+
+
+def __getattr__(name: str):
+ if name in ("_unquotepath", "_safe_chars", "parse_url", *_public_w3lib_objects):
+ obj_type = "attribute" if name == "_safe_chars" else "function"
+ warnings.warn(
+ f"The scrapy.utils.url.{name} {obj_type} is deprecated, use w3lib.url.{name} instead.",
+ ScrapyDeprecationWarning,
+ )
+ return getattr(import_module("w3lib.url"), name)
+
+ raise AttributeError
-from scrapy.utils.python import to_unicode
if TYPE_CHECKING:
from collections.abc import Iterable
from scrapy import Spider
-
UrlT = Union[str, bytes, ParseResult]
def url_is_from_any_domain(url: UrlT, domains: Iterable[str]) -> bool:
"""Return True if the url belongs to any of the given domains"""
- host = parse_url(url).netloc.lower()
+ host = _parse_url(url).netloc.lower()
if not host:
return False
domains = [d.lower() for d in domains]
@@ -46,21 +57,10 @@ def url_is_from_spider(url: UrlT, spider: type[Spider]) -> bool:
def url_has_any_extension(url: UrlT, extensions: Iterable[str]) -> bool:
"""Return True if the url ends with one of the extensions provided"""
- lowercase_path = parse_url(url).path.lower()
+ lowercase_path = _parse_url(url).path.lower()
return any(lowercase_path.endswith(ext) for ext in extensions)
-def parse_url( # pylint: disable=function-redefined
- url: UrlT, encoding: str | None = None
-) -> ParseResult:
- """Return urlparsed url from the given argument (which could be an already
- parsed url)
- """
- if isinstance(url, ParseResult):
- return url
- return cast(ParseResult, urlparse(to_unicode(url, encoding)))
-
-
def escape_ajax(url: str) -> str:
"""
Return the crawlable url
@@ -86,7 +86,7 @@ def escape_ajax(url: str) -> str:
defrag, frag = urldefrag(url)
if not frag.startswith("!"):
return url
- return add_or_replace_parameter(defrag, "_escaped_fragment_", frag[1:])
+ return _add_or_replace_parameter(defrag, "_escaped_fragment_", frag[1:])
def add_http_if_no_scheme(url: str) -> str:
@@ -146,7 +146,7 @@ def guess_scheme(url: str) -> str:
"""Add an URL scheme if missing: file:// for filepath-like input or
http:// otherwise."""
if _is_filesystem_path(url):
- return any_to_uri(url)
+ return _any_to_uri(url)
return add_http_if_no_scheme(url)
diff --git a/tests/test_utils_url.py b/tests/test_utils_url.py
index a15ad749d..62e2b5c1e 100644
--- a/tests/test_utils_url.py
+++ b/tests/test_utils_url.py
@@ -1,10 +1,14 @@
import unittest
+import warnings
+
+import pytest
from scrapy.linkextractors import IGNORED_EXTENSIONS
from scrapy.spiders import Spider
from scrapy.utils.misc import arg_to_iter
from scrapy.utils.url import (
_is_filesystem_path,
+ _public_w3lib_objects,
add_http_if_no_scheme,
guess_scheme,
strip_url,
@@ -607,5 +611,26 @@ class IsPathTestCase(unittest.TestCase):
)
+ "obj_name",
+ [
+ "_unquotepath",
+ "_safe_chars",
+ "parse_url",
+ *_public_w3lib_objects,
+ ],
+)
+def test_deprecated_imports_from_w3lib(obj_name):
+ with warnings.catch_warnings(record=True) as warns:
+ obj_type = "attribute" if obj_name == "_safe_chars" else "function"
+ message = f"The scrapy.utils.url.{obj_name} {obj_type} is deprecated, use w3lib.url.{obj_name} instead."
+
+ from importlib import import_module
+
+ getattr(import_module("scrapy.utils.url"), obj_name)
+
+ assert message in warns[0].message.args
+
+
if __name__ == "__main__":
unittest.main()